Connection Classification

This is an R Markdown Notebook. When you execute code within the notebook, the results appear beneath the code.

Try executing this chunk by clicking the Run button within the chunk or by placing your cursor inside it and pressing Ctrl+Shift+Enter.

suppressMessages(library(tidyverse))
suppressMessages(library(stringr))
suppressMessages(library(ISLR))
suppressMessages(library(caret))
suppressMessages(library(doMC))
suppressMessages(library(plotly))
registerDoMC(cores=4)

Add a new chunk by clicking the Insert Chunk button on the toolbar or by pressing Ctrl+Alt+I.

When you save the notebook, an HTML file containing the code and output will be saved alongside it (click the Preview button or press Ctrl+Shift+K to preview the HTML file).

Getting and proccesing the data

library(stringr)
myData = read.csv('./datasets/data_all_result.txt', stringsAsFactors = F, sep = ' ')
#Create data backup
myData.bkup <- myData
#Create new column: length of model, and number of periodicity, duration and size characteristic in the model.
myData = myData %>% mutate(letter_count = nchar(State))
#Periodicity
myData = myData %>% mutate(strong_p = str_count(State,'[a-i]'))
myData = myData %>% mutate(weak_p = str_count(State,'[A-I]'))
myData = myData %>% mutate(weak_np = str_count(State,'[r-z]'))
myData = myData %>% mutate(strong_np = str_count(State,'[R-Z]'))
#Duration
myData = myData %>% mutate(duration_s = str_count(State,'(a|A|r|R|1|d|D|u|U|4|g|G|x|X|7)'))
myData = myData %>% mutate(duration_m = str_count(State,'(b|B|s|S|2|e|E|v|V|5|h|H|y|Y|8)'))
myData = myData %>% mutate(duration_l = str_count(State,'(c|C|t|T|3|f|F|w|W|6|i|I|z|Z|9)'))
#Size
myData = myData %>% mutate(size_s = str_count(State,'[a-c]') + str_count(State,'[A-C]') + str_count(State,'[r-t]') + str_count(State,'[R-T]') + str_count(State,'[1-3]'))
myData = myData %>% mutate(size_m = str_count(State,'[d-f]') + str_count(State,'[D-F]') + str_count(State,'[u-w]') + str_count(State,'[U-W]') + str_count(State,'[4-6]'))
myData = myData %>% mutate(size_l = str_count(State,'[g-i]') + str_count(State,'[G-I]') + str_count(State,'[x-z]') + str_count(State,'[X-Z]') + str_count(State,'[7-9]'))
#Remove from LabelName unnecessary characters (ej: V42, -17)
myData <- myData %>% mutate(LabelName = gsub('V[0-9]+-','',LabelName))
myData <- myData %>% mutate(LabelName = gsub('-[0-9]+','',LabelName))
myData <- myData %>% mutate(LabelName = gsub('CC[0-9]+-','CC-',LabelName))
#Keep only connection with more than 3 symbols
myData <- myData %>% filter(letter_count > 3)
#Periodicity %
myData <- myData %>% mutate(strong_p = (strong_p / letter_count))
myData <- myData %>% mutate(weak_p = (weak_p / letter_count))
myData <- myData %>% mutate(strong_np = (strong_np / letter_count))
myData <- myData %>% mutate(weak_np = (weak_np / letter_count))
#Duration %
myData <- myData %>% mutate(duration_s = (duration_s / letter_count))
myData <- myData %>% mutate(duration_m = (duration_m / letter_count))
myData <- myData %>% mutate(duration_l = (duration_l / letter_count))
#Size %
myData <- myData %>% mutate(size_s = (size_s / letter_count))
myData <- myData %>% mutate(size_m = (size_m / letter_count))
myData <- myData %>% mutate(size_l = (size_l / letter_count))
#head(myData)
myData[1:20,]

#Making feature vectors
feature_vectors = myData[,c('strong_p','weak_p','weak_np','strong_np','duration_s','duration_m','duration_l','size_s','size_m','size_l','letter_count','Label','LabelName','port','proto')]
names(feature_vectors) = c("sp","wp","wnp","snp","ds","dm","dl","ss","sm","sl","length","class","subclass","port","proto")
feature_vectors$class = factor(feature_vectors$class)
feature_vectors$subclass = factor(feature_vectors$subclass)
feature_vectors$proto = factor(feature_vectors$proto)

Create training set and testset

nrow(data_train %>% filter(class == 'Normal'))

[1] 2278

Random Forest Classificator

  # Random Forest
rfFit <- train(class ~ sp+wp+wnp+snp+ds+dm+dl+ss+sm+sl,
               data = training,
               metric="ROC",
               method = "rf",
               trControl = ctrl_fast)

Aggregating results
Selecting tuning parameters
Fitting mtry = 2 on full training set

rfFit

Random Forest 

10256 samples
   10 predictors
    2 classes: 'Botnet', 'Normal' 

No pre-processing
Resampling: Cross-Validated (10 fold) 
Summary of sample sizes: 9231, 9231, 9230, 9231, 9230, 9231, ... 
Resampling results across tuning parameters:

  mtry  ROC        Sens       Spec     
   2    0.9760942  0.9204366  0.9680167
   6    0.9750400  0.9253103  0.9658713
  10    0.9746732  0.9233598  0.9647010

ROC was used to select the optimal model using  the largest value.
The final value used for the model was mtry = 2. 

rfFit$finalModel

Call:
 randomForest(x = x, y = y, mtry = param$mtry) 
               Type of random forest: classification
                     Number of trees: 500
No. of variables tried at each split: 2

        OOB estimate of  error rate: 5.39%
Confusion matrix:
       Botnet Normal class.error
Botnet   4718    410  0.07995320
Normal    143   4985  0.02788612

a$overall

      Accuracy          Kappa  AccuracyLower  AccuracyUpper   AccuracyNull AccuracyPValue 
  9.168017e-01   8.098174e-01   9.032790e-01   9.289854e-01   6.925986e-01  2.237885e-121 
 McnemarPValue 
  9.335163e-06 

library(ggplot2)
library(plotROC)
selectedIndices <- rfFit$pred$mtry == 2
ggplot(cbind(predsrfprobs,class=testing$class), 
       aes(m = Botnet, d = factor(class, labels=c("Normal","Botnet"),levels = c("Normal", "Botnet")))) + 
    geom_roc(hjust = -0.4, vjust = 1.5,colour='orange') + 
  theme_bw()

cbind(predsrfprobs,class=testing$class)

KNN

#Checking distibution in origanl data and partitioned data
prop.table(table(training$class)) * 100

Botnet Normal 
    50     50 

prop.table(table(testing$class)) * 100

  Botnet   Normal 
69.25986 30.74014 

prop.table(table(feature_vectors$class)) * 100

 Botnet  Normal 
69.2449 30.7551 

trainX <- training[,names(training) != "class"]
preProcValues <- preProcess(x = trainX,method = c("center", "scale"))
preProcValues

Created from 10244 samples and 13 variables

Pre-processing:
  - centered (11)
  - ignored (2)
  - scaled (11)

knnFit <- train(class ~ sp+wp+wnp+snp+ds+dm+dl+ss+sm+sl, data = training, method = "knn", trControl = ctrl_fast, preProcess = c("center","scale"), tuneLength = 20)

The metric "Accuracy" was not in the result set. ROC will be used instead.

Aggregating results
Selecting tuning parameters
Fitting k = 15 on full training set

#Output of kNN fit
knnFit

k-Nearest Neighbors 

10256 samples
   10 predictors
    2 classes: 'Botnet', 'Normal' 

Pre-processing: centered (10), scaled (10) 
Resampling: Cross-Validated (10 fold) 
Summary of sample sizes: 9230, 9230, 9230, 9231, 9231, 9230, ... 
Resampling results across tuning parameters:

  k   ROC        Sens       Spec     
   5  0.9660665  0.8738304  0.9450079
   7  0.9679714  0.8753910  0.9432520
   9  0.9678652  0.8689567  0.9413031
  11  0.9679040  0.8670051  0.9432501
  13  0.9680515  0.8625221  0.9418856
  15  0.9680761  0.8654442  0.9416903
  17  0.9678730  0.8595954  0.9413004
  19  0.9673341  0.8603752  0.9405188
  21  0.9668209  0.8605686  0.9393492
  23  0.9661648  0.8576454  0.9381792
  25  0.9656652  0.8549156  0.9397394
  27  0.9654901  0.8508224  0.9403235
  29  0.9652638  0.8498458  0.9368151
  31  0.9647511  0.8486747  0.9356440
  33  0.9644415  0.8510154  0.9336950
  35  0.9640843  0.8477004  0.9350588
  37  0.9639755  0.8475070  0.9350599
  39  0.9636649  0.8453620  0.9348650
  41  0.9633665  0.8453616  0.9334989
  43  0.9631422  0.8451655  0.9331087

ROC was used to select the optimal model using  the largest value.
The final value used for the model was k = 15. 

#Plotting yields Number of Neighbours Vs accuracy (based on repeated cross validation)
plot(knnFit)

knnPredict <- predict(knnFit,newdata = testing )
#Get the confusion matrix to see accuracy value and other parameter values
confusionMatrix(knnPredict, testing$class )

Confusion Matrix and Statistics

          Reference
Prediction Botnet Normal
    Botnet   1121     40
    Normal    161    529
                                          
               Accuracy : 0.8914          
                 95% CI : (0.8763, 0.9052)
    No Information Rate : 0.6926          
    P-Value [Acc > NIR] : < 2.2e-16       
                                          
                  Kappa : 0.7592          
 Mcnemar's Test P-Value : < 2.2e-16       
                                          
            Sensitivity : 0.8744          
            Specificity : 0.9297          
         Pos Pred Value : 0.9655          
         Neg Pred Value : 0.7667          
             Prevalence : 0.6926          
         Detection Rate : 0.6056          
   Detection Prevalence : 0.6272          
      Balanced Accuracy : 0.9021          
                                          
       'Positive' Class : Botnet          
                                          

mean(knnPredict == testing$class)

[1] 0.89141

library(pROC)
knnPredict <- predict(knnFit,newdata = testing , type="prob")
knnROC <- roc(testing$class,knnPredict[,"Botnet"], levels = c('Normal','Botnet'))#rev(testing$class))
knnROC

Call:
roc.default(response = testing$class, predictor = knnPredict[,     "Botnet"], levels = c("Normal", "Botnet"))

Data: knnPredict[, "Botnet"] in 569 controls (testing$class Normal) < 1282 cases (testing$class Botnet).
Area under the curve: 0.9644

ggplot(cbind(knnPredict,class=testing$class), 
       aes(m = Botnet, d = factor(class, labels=c("Normal","Botnet"),levels = c("Normal", "Botnet")))) + 
    geom_roc(hjust = -0.4, vjust = 1.5,colour='orange') + 
  theme_bw()

#plot(knnROC, type="S", print.thres= 0.5)

Logistic Regression

logicRFit <- train(class ~ sp+wp+wnp+snp+ds+dm+dl+ss+sm+sl, method='glm', trControl = ctrl_fast,preProcess=c('scale', 'center'), data=training, family=binomial(link='logit'))

The metric "Accuracy" was not in the result set. ROC will be used instead.prediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleading

Aggregating results
Fitting final model on full training set

#logicRFit <- train(class ~ sp*wp*wnp*snp*ds*dm*dl*ss*sm*sl, method='glm', trControl = ctrl_fast,preProcess=c('scale', 'center'), data=training, family=binomial(link='logit'))
#logicRFit <- train(class ~ sp+wp+wnp+snp+ds+dm+dl+ss+sm+sl, method='glm', trControl = ctrl_fast,preProcess=c('scale', 'center'), data=training, family=binomial(link='logit'))
#summary(logicRFit)
#Output of Logistic Regression fit
logicRFit

Generalized Linear Model 

10256 samples
   10 predictors
    2 classes: 'Botnet', 'Normal' 

Pre-processing: scaled (10), centered (10) 
Resampling: Cross-Validated (10 fold) 
Summary of sample sizes: 9230, 9230, 9232, 9230, 9231, 9230, ... 
Resampling results:

  ROC        Sens       Spec     
  0.9401934  0.8373648  0.9167341

 

logicRPredict <- predict(logicRFit, newdata = testing )

prediction from a rank-deficient fit may be misleading

confusionMatrix(logicRPredict, testing$class)

Confusion Matrix and Statistics

          Reference
Prediction Botnet Normal
    Botnet   1083     45
    Normal    199    524
                                          
               Accuracy : 0.8682          
                 95% CI : (0.8519, 0.8833)
    No Information Rate : 0.6926          
    P-Value [Acc > NIR] : < 2.2e-16       
                                          
                  Kappa : 0.7121          
 Mcnemar's Test P-Value : < 2.2e-16       
                                          
            Sensitivity : 0.8448          
            Specificity : 0.9209          
         Pos Pred Value : 0.9601          
         Neg Pred Value : 0.7248          
             Prevalence : 0.6926          
         Detection Rate : 0.5851          
   Detection Prevalence : 0.6094          
      Balanced Accuracy : 0.8828          
                                          
       'Positive' Class : Botnet          
                                          

logicRPredict <- predict(logicRFit, newdata = testing, type="prob")

prediction from a rank-deficient fit may be misleading

logicROC <- roc(testing$class,logicRPredict[,"Botnet"], levels = c('Normal','Botnet'))#rev(testing$class))
ggplot(cbind(logicRPredict,class=testing$class), 
       aes(m = Botnet, d = factor(class, labels=c("Normal","Botnet"),levels = c("Normal", "Botnet")))) + 
    geom_roc(hjust = -0.4, vjust = 1.5,colour='orange') + 
  theme_bw()

#logicROC

Naive Bayes

naiveBayesFit <- train(class ~ sp+wp+wnp+snp+ds+dm+dl+ss+sm+sl, method='nb', trControl = ctrl_fast,preProcess=c('scale', 'center'), data=training)

The metric "Accuracy" was not in the result set. ROC will be used instead.Numerical 0 probability for all classes with observation 13Numerical 0 probability for all classes with observation 3Numerical 0 probability for all classes with observation 24Numerical 0 probability for all classes with observation 55Numerical 0 probability for all classes with observation 86Numerical 0 probability for all classes with observation 101Numerical 0 probability for all classes with observation 120Numerical 0 probability for all classes with observation 129Numerical 0 probability for all classes with observation 130Numerical 0 probability for all classes with observation 134Numerical 0 probability for all classes with observation 137Numerical 0 probability for all classes with observation 80Numerical 0 probability for all classes with observation 155Numerical 0 probability for all classes with observation 181Numerical 0 probability for all classes with observation 125Numerical 0 probability for all classes with observation 139Numerical 0 probability for all classes with observation 219Numerical 0 probability for all classes with observation 142Numerical 0 probability for all classes with observation 160Numerical 0 probability for all classes with observation 253Numerical 0 probability for all classes with observation 274Numerical 0 probability for all classes with observation 283Numerical 0 probability for all classes with observation 208Numerical 0 probability for all classes with observation 217Numerical 0 probability for all classes with observation 221Numerical 0 probability for all classes with observation 228Numerical 0 probability for all classes with observation 230Numerical 0 probability for all classes with observation 348Numerical 0 probability for all classes with observation 349Numerical 0 probability for all classes with observation 354Numerical 0 probability for all classes with observation 355Numerical 0 probability for all classes with observation 261Numerical 0 probability for all classes with observation 378Numerical 0 probability for all classes with observation 379Numerical 0 probability for all classes with observation 265Numerical 0 probability for all classes with observation 278Numerical 0 probability for all classes with observation 396Numerical 0 probability for all classes with observation 291Numerical 0 probability for all classes with observation 406Numerical 0 probability for all classes with observation 295Numerical 0 probability for all classes with observation 411Numerical 0 probability for all classes with observation 420Numerical 0 probability for all classes with observation 308Numerical 0 probability for all classes with observation 422Numerical 0 probability for all classes with observation 437Numerical 0 probability for all classes with observation 330Numerical 0 probability for all classes with observation 348Numerical 0 probability for all classes with observation 465Numerical 0 probability for all classes with observation 367Numerical 0 probability for all classes with observation 374Numerical 0 probability for all classes with observation 489Numerical 0 probability for all classes with observation 494Numerical 0 probability for all classes with observation 381Numerical 0 probability for all classes with observation 529Numerical 0 probability for all classes with observation 399Numerical 0 probability for all classes with observation 531Numerical 0 probability for all classes with observation 410Numerical 0 probability for all classes with observation 545Numerical 0 probability for all classes with observation 546Numerical 0 probability for all classes with observation 547Numerical 0 probability for all classes with observation 431Numerical 0 probability for all classes with observation 551Numerical 0 probability for all classes with observation 442Numerical 0 probability for all classes with observation 573Numerical 0 probability for all classes with observation 575Numerical 0 probability for all classes with observation 576Numerical 0 probability for all classes with observation 577Numerical 0 probability for all classes with observation 578Numerical 0 probability for all classes with observation 460Numerical 0 probability for all classes with observation 462Numerical 0 probability for all classes with observation 617Numerical 0 probability for all classes with observation 471Numerical 0 probability for all classes with observation 635Numerical 0 probability for all classes with observation 638Numerical 0 probability for all classes with observation 643Numerical 0 probability for all classes with observation 493Numerical 0 probability for all classes with observation 657Numerical 0 probability for all classes with observation 658Numerical 0 probability for all classes with observation 673Numerical 0 probability for all classes with observation 511Numerical 0 probability for all classes with observation 513Numerical 0 probability for all classes with observation 696Numerical 0 probability for all classes with observation 697Numerical 0 probability for all classes with observation 534Numerical 0 probability for all classes with observation 707Numerical 0 probability for all classes with observation 538Numerical 0 probability for all classes with observation 721Numerical 0 probability for all classes with observation 743Numerical 0 probability for all classes with observation 546Numerical 0 probability for all classes with observation 547Numerical 0 probability for all classes with observation 553Numerical 0 probability for all classes with observation 784Numerical 0 probability for all classes with observation 787Numerical 0 probability for all classes with observation 791Numerical 0 probability for all classes with observation 576Numerical 0 probability for all classes with observation 802Numerical 0 probability for all classes with observation 805Numerical 0 probability for all classes with observation 806Numerical 0 probability for all classes with observation 810Numerical 0 probability for all classes with observation 596Numerical 0 probability for all classes with observation 830Numerical 0 probability for all classes with observation 834Numerical 0 probability for all classes with observation 267Numerical 0 probability for all classes with observation 836Numerical 0 probability for all classes with observation 621Numerical 0 probability for all classes with observation 627Numerical 0 probability for all classes with observation 634Numerical 0 probability for all classes with observation 643Numerical 0 probability for all classes with observation 876Numerical 0 probability for all classes with observation 883Numerical 0 probability for all classes with observation 651Numerical 0 probability for all classes with observation 884Numerical 0 probability for all classes with observation 659Numerical 0 probability for all classes with observation 669Numerical 0 probability for all classes with observation 915Numerical 0 probability for all classes with observation 924Numerical 0 probability for all classes with observation 686Numerical 0 probability for all classes with observation 927Numerical 0 probability for all classes with observation 691Numerical 0 probability for all classes with observation 693Numerical 0 probability for all classes with observation 702Numerical 0 probability for all classes with observation 954Numerical 0 probability for all classes with observation 956Numerical 0 probability for all classes with observation 960Numerical 0 probability for all classes with observation 721Numerical 0 probability for all classes with observation 727Numerical 0 probability for all classes with observation 984Numerical 0 probability for all classes with observation 987Numerical 0 probability for all classes with observation 737Numerical 0 probability for all classes with observation 994Numerical 0 probability for all classes with observation 1003Numerical 0 probability for all classes with observation 1005Numerical 0 probability for all classes with observation 1010Numerical 0 probability for all classes with observation 759Numerical 0 probability for all classes with observation 1015Numerical 0 probability for all classes with observation 763Numerical 0 probability for all classes with observation 765Numerical 0 probability for all classes with observation 769Numerical 0 probability for all classes with observation 787Numerical 0 probability for all classes with observation 789Numerical 0 probability for all classes with observation 799Numerical 0 probability for all classes with observation 811Numerical 0 probability for all classes with observation 13Numerical 0 probability for all classes with observation 829Numerical 0 probability for all classes with observation 833Numerical 0 probability for all classes with observation 55Numerical 0 probability for all classes with observation 872Numerical 0 probability for all classes with observation 880Numerical 0 probability for all classes with observation 86Numerical 0 probability for all classes with observation 101Numerical 0 probability for all classes with observation 898Numerical 0 probability for all classes with observation 902Numerical 0 probability for all classes with observation 120Numerical 0 probability for all classes with observation 917Numerical 0 probability for all classes with observation 129Numerical 0 probability for all classes with observation 130Numerical 0 probability for all classes with observation 134Numerical 0 probability for all classes with observation 925Numerical 0 probability for all classes with observation 137Numerical 0 probability for all classes with observation 927Numerical 0 probability for all classes with observation 938Numerical 0 probability for all classes with observation 155Numerical 0 probability for all classes with observation 949Numerical 0 probability for all classes with observation 181Numerical 0 probability for all classes with observation 965Numerical 0 probability for all classes with observation 219Numerical 0 probability for all classes with observation 989Numerical 0 probability for all classes with observation 993Numerical 0 probability for all classes with observation 1003Numerical 0 probability for all classes with observation 1008Numerical 0 probability for all classes with observation 253Numerical 0 probability for all classes with observation 1018Numerical 0 probability for all classes with observation 274Numerical 0 probability for all classes with observation 283Numerical 0 probability for all classes with observation 348Numerical 0 probability for all classes with observation 349Numerical 0 probability for all classes with observation 3Numerical 0 probability for all classes with observation 354Numerical 0 probability for all classes with observation 355Numerical 0 probability for all classes with observation 24Numerical 0 probability for all classes with observation 378Numerical 0 probability for all classes with observation 379Numerical 0 probability for all classes with observation 396Numerical 0 probability for all classes with observation 406Numerical 0 probability for all classes with observation 411Numerical 0 probability for all classes with observation 420Numerical 0 probability for all classes with observation 422Numerical 0 probability for all classes with observation 437Numerical 0 probability for all classes with observation 80Numerical 0 probability for all classes with observation 125Numerical 0 probability for all classes with observation 465Numerical 0 probability for all classes with observation 139Numerical 0 probability for all classes with observation 142Numerical 0 probability for all classes with observation 489Numerical 0 probability for all classes with observation 494Numerical 0 probability for all classes with observation 160Numerical 0 probability for all classes with observation 529Numerical 0 probability for all classes with observation 531Numerical 0 probability for all classes with observation 545Numerical 0 probability for all classes with observation 546Numerical 0 probability for all classes with observation 547Numerical 0 probability for all classes with observation 551Numerical 0 probability for all classes with observation 208Numerical 0 probability for all classes with observation 217Numerical 0 probability for all classes with observation 573Numerical 0 probability for all classes with observation 575Numerical 0 probability for all classes with observation 221Numerical 0 probability for all classes with observation 576Numerical 0 probability for all classes with observation 577Numerical 0 probability for all classes with observation 578Numerical 0 probability for all classes with observation 228Numerical 0 probability for all classes with observation 230Numerical 0 probability for all classes with observation 617Numerical 0 probability for all classes with observation 261Numerical 0 probability for all classes with observation 265Numerical 0 probability for all classes with observation 635Numerical 0 probability for all classes with observation 638Numerical 0 probability for all classes with observation 643Numerical 0 probability for all classes with observation 278Numerical 0 probability for all classes with observation 657Numerical 0 probability for all classes with observation 658Numerical 0 probability for all classes with observation 291Numerical 0 probability for all classes with observation 295Numerical 0 probability for all classes with observation 673Numerical 0 probability for all classes with observation 308Numerical 0 probability for all classes with observation 696Numerical 0 probability for all classes with observation 697Numerical 0 probability for all classes with observation 707Numerical 0 probability for all classes with observation 330Numerical 0 probability for all classes with observation 546Numerical 0 probability for all classes with observation 721Numerical 0 probability for all classes with observation 348Numerical 0 probability for all classes with observation 743Numerical 0 probability for all classes with observation 367Numerical 0 probability for all classes with observation 374Numerical 0 probability for all classes with observation 381Numerical 0 probability for all classes with observation 784Numerical 0 probability for all classes with observation 787Numerical 0 probability for all classes with observation 791Numerical 0 probability for all classes with observation 399Numerical 0 probability for all classes with observation 802Numerical 0 probability for all classes with observation 805Numerical 0 probability for all classes with observation 806Numerical 0 probability for all classes with observation 410Numerical 0 probability for all classes with observation 810Numerical 0 probability for all classes with observation 830Numerical 0 probability for all classes with observation 431Numerical 0 probability for all classes with observation 834Numerical 0 probability for all classes with observation 836Numerical 0 probability for all classes with observation 442Numerical 0 probability for all classes with observation 876Numerical 0 probability for all classes with observation 883Numerical 0 probability for all classes with observation 884Numerical 0 probability for all classes with observation 460Numerical 0 probability for all classes with observation 462Numerical 0 probability for all classes with observation 915Numerical 0 probability for all classes with observation 471Numerical 0 probability for all classes with observation 924Numerical 0 probability for all classes with observation 927Numerical 0 probability for all classes with observation 493Numerical 0 probability for all classes with observation 511Numerical 0 probability for all classes with observation 513Numerical 0 probability for all classes with observation 954Numerical 0 probability for all classes with observation 956Numerical 0 probability for all classes with observation 960Numerical 0 probability for all classes with observation 534Numerical 0 probability for all classes with observation 538Numerical 0 probability for all classes with observation 546Numerical 0 probability for all classes with observation 547Numerical 0 probability for all classes with observation 984Numerical 0 probability for all classes with observation 987Numerical 0 probability for all classes with observation 553Numerical 0 probability for all classes with observation 994Numerical 0 probability for all classes with observation 1003Numerical 0 probability for all classes with observation 1005Numerical 0 probability for all classes with observation 1010Numerical 0 probability for all classes with observation 576Numerical 0 probability for all classes with observation 1015Numerical 0 probability for all classes with observation 596Numerical 0 probability for all classes with observation 621Numerical 0 probability for all classes with observation 627Numerical 0 probability for all classes with observation 634Numerical 0 probability for all classes with observation 643Numerical 0 probability for all classes with observation 651Numerical 0 probability for all classes with observation 659Numerical 0 probability for all classes with observation 669Numerical 0 probability for all classes with observation 508Numerical 0 probability for all classes with observation 686Numerical 0 probability for all classes with observation 691Numerical 0 probability for all classes with observation 693Numerical 0 probability for all classes with observation 702Numerical 0 probability for all classes with observation 721Numerical 0 probability for all classes with observation 727Numerical 0 probability for all classes with observation 737Numerical 0 probability for all classes with observation 759Numerical 0 probability for all classes with observation 763Numerical 0 probability for all classes with observation 34Numerical 0 probability for all classes with observation 765Numerical 0 probability for all classes with observation 769Numerical 0 probability for all classes with observation 42Numerical 0 probability for all classes with observation 46Numerical 0 probability for all classes with observation 787Numerical 0 probability for all classes with observation 789Numerical 0 probability for all classes with observation 799Numerical 0 probability for all classes with observation 811Numerical 0 probability for all classes with observation 98Numerical 0 probability for all classes with observation 829Numerical 0 probability for all classes with observation 124Numerical 0 probability for all classes with observation 833Numerical 0 probability for all classes with observation 135Numerical 0 probability for all classes with observation 872Numerical 0 probability for all classes with observation 176Numerical 0 probability for all classes with observation 880Numerical 0 probability for all classes with observation 192Numerical 0 probability for all classes with observation 898Numerical 0 probability for all classes with observation 902Numerical 0 probability for all classes with observation 572Numerical 0 probability for all classes with observation 232Numerical 0 probability for all classes with observation 917Numerical 0 probability for all classes with observation 234Numerical 0 probability for all classes with observation 925Numerical 0 probability for all classes with observation 927Numerical 0 probability for all classes with observation 258Numerical 0 probability for all classes with observation 938Numerical 0 probability for all classes with observation 295Numerical 0 probability for all classes with observation 296Numerical 0 probability for all classes with observation 298Numerical 0 probability for all classes with observation 300Numerical 0 probability for all classes with observation 949Numerical 0 probability for all classes with observation 306Numerical 0 probability for all classes with observation 965Numerical 0 probability for all classes with observation 324Numerical 0 probability for all classes with observation 989Numerical 0 probability for all classes with observation 993Numerical 0 probability for all classes with observation 1003Numerical 0 probability for all classes with observation 1008Numerical 0 probability for all classes with observation 359Numerical 0 probability for all classes with observation 1018Numerical 0 probability for all classes with observation 369Numerical 0 probability for all classes with observation 375Numerical 0 probability for all classes with observation 380Numerical 0 probability for all classes with observation 390Numerical 0 probability for all classes with observation 404Numerical 0 probability for all classes with observation 425Numerical 0 probability for all classes with observation 483Numerical 0 probability for all classes with observation 487Numerical 0 probability for all classes with observation 508Numerical 0 probability for all classes with observation 513Numerical 0 probability for all classes with observation 546Numerical 0 probability for all classes with observation 550Numerical 0 probability for all classes with observation 603Numerical 0 probability for all classes with observation 613Numerical 0 probability for all classes with observation 619Numerical 0 probability for all classes with observation 630Numerical 0 probability for all classes with observation 632Numerical 0 probability for all classes with observation 639Numerical 0 probability for all classes with observation 643Numerical 0 probability for all classes with observation 32Numerical 0 probability for all classes with observation 658Numerical 0 probability for all classes with observation 44Numerical 0 probability for all classes with observation 47Numerical 0 probability for all classes with observation 50Numerical 0 probability for all classes with observation 68Numerical 0 probability for all classes with observation 707Numerical 0 probability for all classes with observation 711Numerical 0 probability for all classes with observation 747Numerical 0 probability for all classes with observation 750Numerical 0 probability for all classes with observation 116Numerical 0 probability for all classes with observation 759Numerical 0 probability for all classes with observation 118Numerical 0 probability for all classes with observation 760Numerical 0 probability for all classes with observation 768Numerical 0 probability for all classes with observation 132Numerical 0 probability for all classes with observation 150Numerical 0 probability for all classes with observation 803Numerical 0 probability for all classes with observation 839Numerical 0 probability for all classes with observation 869Numerical 0 probability for all classes with observation 883Numerical 0 probability for all classes with observation 253Numerical 0 probability for all classes with observation 254Numerical 0 probability for all classes with observation 256Numerical 0 probability for all classes with observation 902Numerical 0 probability for all classes with observation 268Numerical 0 probability for all classes with observation 923Numerical 0 probability for all classes with observation 927Numerical 0 probability for all classes with observation 932Numerical 0 probability for all classes with observation 287Numerical 0 probability for all classes with observation 936Numerical 0 probability for all classes with observation 292Numerical 0 probability for all classes with observation 962Numerical 0 probability for all classes with observation 980Numerical 0 probability for all classes with observation 331Numerical 0 probability for all classes with observation 333Numerical 0 probability for all classes with observation 1007Numerical 0 probability for all classes with observation 346Numerical 0 probability for all classes with observation 360Numerical 0 probability for all classes with observation 368Numerical 0 probability for all classes with observation 373Numerical 0 probability for all classes with observation 34Numerical 0 probability for all classes with observation 42Numerical 0 probability for all classes with observation 46Numerical 0 probability for all classes with observation 98Numerical 0 probability for all classes with observation 124Numerical 0 probability for all classes with observation 467Numerical 0 probability for all classes with observation 471Numerical 0 probability for all classes with observation 472Numerical 0 probability for all classes with observation 135Numerical 0 probability for all classes with observation 481Numerical 0 probability for all classes with observation 176Numerical 0 probability for all classes with observation 512Numerical 0 probability for all classes with observation 192Numerical 0 probability for all classes with observation 524Numerical 0 probability for all classes with observation 532Numerical 0 probability for all classes with observation 546Numerical 0 probability for all classes with observation 232Numerical 0 probability for all classes with observation 234Numerical 0 probability for all classes with observation 554Numerical 0 probability for all classes with observation 559Numerical 0 probability for all classes with observation 560Numerical 0 probability for all classes with observation 583Numerical 0 probability for all classes with observation 586Numerical 0 probability for all classes with observation 258Numerical 0 probability for all classes with observation 295Numerical 0 probability for all classes with observation 296Numerical 0 probability for all classes with observation 298Numerical 0 probability for all classes with observation 300Numerical 0 probability for all classes with observation 630Numerical 0 probability for all classes with observation 306Numerical 0 probability for all classes with observation 324Numerical 0 probability for all classes with observation 659Numerical 0 probability for all classes with observation 660Numerical 0 probability for all classes with observation 672Numerical 0 probability for all classes with observation 359Numerical 0 probability for all classes with observation 369Numerical 0 probability for all classes with observation 684Numerical 0 probability for all classes with observation 375Numerical 0 probability for all classes with observation 380Numerical 0 probability for all classes with observation 390Numerical 0 probability for all classes with observation 713Numerical 0 probability for all classes with observation 404Numerical 0 probability for all classes with observation 716Numerical 0 probability for all classes with observation 425Numerical 0 probability for all classes with observation 740Numerical 0 probability for all classes with observation 775Numerical 0 probability for all classes with observation 783Numerical 0 probability for all classes with observation 786Numerical 0 probability for all classes with observation 787Numerical 0 probability for all classes with observation 483Numerical 0 probability for all classes with observation 487Numerical 0 probability for all classes with observation 794Numerical 0 probability for all classes with observation 803Numerical 0 probability for all classes with observation 508Numerical 0 probability for all classes with observation 513Numerical 0 probability for all classes with observation 546Numerical 0 probability for all classes with observation 550Numerical 0 probability for all classes with observation 847Numerical 0 probability for all classes with observation 868Numerical 0 probability for all classes with observation 603Numerical 0 probability for all classes with observation 874Numerical 0 probability for all classes with observation 613Numerical 0 probability for all classes with observation 883Numerical 0 probability for all classes with observation 888Numerical 0 probability for all classes with observation 619Numerical 0 probability for all classes with observation 630Numerical 0 probability for all classes with observation 902Numerical 0 probability for all classes with observation 632Numerical 0 probability for all classes with observation 905Numerical 0 probability for all classes with observation 639Numerical 0 probability for all classes with observation 912Numerical 0 probability for all classes with observation 643Numerical 0 probability for all classes with observation 927Numerical 0 probability for all classes with observation 658Numerical 0 probability for all classes with observation 952Numerical 0 probability for all classes with observation 968Numerical 0 probability for all classes with observation 977Numerical 0 probability for all classes with observation 707Numerical 0 probability for all classes with observation 711Numerical 0 probability for all classes with observation 994Numerical 0 probability for all classes with observation 1024Numerical 0 probability for all classes with observation 747Numerical 0 probability for all classes with observation 750Numerical 0 probability for all classes with observation 759Numerical 0 probability for all classes with observation 760Numerical 0 probability for all classes with observation 32Numerical 0 probability for all classes with observation 768Numerical 0 probability for all classes with observation 44Numerical 0 probability for all classes with observation 47Numerical 0 probability for all classes with observation 50Numerical 0 probability for all classes with observation 68Numerical 0 probability for all classes with observation 803Numerical 0 probability for all classes with observation 116Numerical 0 probability for all classes with observation 118Numerical 0 probability for all classes with observation 839Numerical 0 probability for all classes with observation 132Numerical 0 probability for all classes with observation 150Numerical 0 probability for all classes with observation 869Numerical 0 probability for all classes with observation 883Numerical 0 probability for all classes with observation 902Numerical 0 probability for all classes with observation 923Numerical 0 probability for all classes with observation 267Numerical 0 probability for all classes with observation 927Numerical 0 probability for all classes with observation 932Numerical 0 probability for all classes with observation 936Numerical 0 probability for all classes with observation 962Numerical 0 probability for all classes with observation 253Numerical 0 probability for all classes with observation 254Numerical 0 probability for all classes with observation 256Numerical 0 probability for all classes with observation 268Numerical 0 probability for all classes with observation 980Numerical 0 probability for all classes with observation 287Numerical 0 probability for all classes with observation 292Numerical 0 probability for all classes with observation 1007Numerical 0 probability for all classes with observation 331Numerical 0 probability for all classes with observation 333Numerical 0 probability for all classes with observation 346Numerical 0 probability for all classes with observation 360Numerical 0 probability for all classes with observation 368Numerical 0 probability for all classes with observation 373Numerical 0 probability for all classes with observation 467Numerical 0 probability for all classes with observation 471Numerical 0 probability for all classes with observation 472Numerical 0 probability for all classes with observation 481Numerical 0 probability for all classes with observation 512Numerical 0 probability for all classes with observation 524Numerical 0 probability for all classes with observation 532Numerical 0 probability for all classes with observation 546Numerical 0 probability for all classes with observation 554Numerical 0 probability for all classes with observation 6Numerical 0 probability for all classes with observation 559Numerical 0 probability for all classes with observation 560Numerical 0 probability for all classes with observation 21Numerical 0 probability for all classes with observation 583Numerical 0 probability for all classes with observation 586Numerical 0 probability for all classes with observation 38Numerical 0 probability for all classes with observation 42Numerical 0 probability for all classes with observation 630Numerical 0 probability for all classes with observation 78Numerical 0 probability for all classes with observation 79Numerical 0 probability for all classes with observation 659Numerical 0 probability for all classes with observation 660Numerical 0 probability for all classes with observation 96Numerical 0 probability for all classes with observation 672Numerical 0 probability for all classes with observation 684Numerical 0 probability for all classes with observation 713Numerical 0 probability for all classes with observation 716Numerical 0 probability for all classes with observation 116Numerical 0 probability for all classes with observation 118Numerical 0 probability for all classes with observation 119Numerical 0 probability for all classes with observation 129Numerical 0 probability for all classes with observation 142Numerical 0 probability for all classes with observation 740Numerical 0 probability for all classes with observation 152Numerical 0 probability for all classes with observation 159Numerical 0 probability for all classes with observation 775Numerical 0 probability for all classes with observation 783Numerical 0 probability for all classes with observation 786Numerical 0 probability for all classes with observation 787Numerical 0 probability for all classes with observation 191Numerical 0 probability for all classes with observation 194Numerical 0 probability for all classes with observation 794Numerical 0 probability for all classes with observation 803Numerical 0 probability for all classes with observation 218Numerical 0 probability for all classes with observation 234Numerical 0 probability for all classes with observation 235Numerical 0 probability for all classes with observation 847Numerical 0 probability for all classes with observation 868Numerical 0 probability for all classes with observation 874Numerical 0 probability for all classes with observation 261Numerical 0 probability for all classes with observation 883Numerical 0 probability for all classes with observation 270Numerical 0 probability for all classes with observation 888Numerical 0 probability for all classes with observation 902Numerical 0 probability for all classes with observation 905Numerical 0 probability for all classes with observation 912Numerical 0 probability for all classes with observation 294Numerical 0 probability for all classes with observation 927Numerical 0 probability for all classes with observation 952Numerical 0 probability for all classes with observation 968Numerical 0 probability for all classes with observation 345Numerical 0 probability for all classes with observation 977Numerical 0 probability for all classes with observation 351Numerical 0 probability for all classes with observation 355Numerical 0 probability for all classes with observation 994Numerical 0 probability for all classes with observation 361Numerical 0 probability for all classes with observation 363Numerical 0 probability for all classes with observation 1024Numerical 0 probability for all classes with observation 424Numerical 0 probability for all classes with observation 449Numerical 0 probability for all classes with observation 450Numerical 0 probability for all classes with observation 470Numerical 0 probability for all classes with observation 471Numerical 0 probability for all classes with observation 472Numerical 0 probability for all classes with observation 485Numerical 0 probability for all classes with observation 502Numerical 0 probability for all classes with observation 503Numerical 0 probability for all classes with observation 546Numerical 0 probability for all classes with observation 7Numerical 0 probability for all classes with observation 538Numerical 0 probability for all classes with observation 549Numerical 0 probability for all classes with observation 32Numerical 0 probability for all classes with observation 560Numerical 0 probability for all classes with observation 567Numerical 0 probability for all classes with observation 62Numerical 0 probability for all classes with observation 65Numerical 0 probability for all classes with observation 70Numerical 0 probability for all classes with observation 583Numerical 0 probability for all classes with observation 584Numerical 0 probability for all classes with observation 85Numerical 0 probability for all classes with observation 97Numerical 0 probability for all classes with observation 106Numerical 0 probability for all classes with observation 595Numerical 0 probability for all classes with observation 117Numerical 0 probability for all classes with observation 621Numerical 0 probability for all classes with observation 124Numerical 0 probability for all classes with observation 625Numerical 0 probability for all classes with observation 626Numerical 0 probability for all classes with observation 628Numerical 0 probability for all classes with observation 629Numerical 0 probability for all classes with observation 641Numerical 0 probability for all classes with observation 643Numerical 0 probability for all classes with observation 154Numerical 0 probability for all classes with observation 160Numerical 0 probability for all classes with observation 657Numerical 0 probability for all classes with observation 660Numerical 0 probability for all classes with observation 197Numerical 0 probability for all classes with observation 199Numerical 0 probability for all classes with observation 200Numerical 0 probability for all classes with observation 229Numerical 0 probability for all classes with observation 237Numerical 0 probability for all classes with observation 244Numerical 0 probability for all classes with observation 722Numerical 0 probability for all classes with observation 745Numerical 0 probability for all classes with observation 293Numerical 0 probability for all classes with observation 765Numerical 0 probability for all classes with observation 306Numerical 0 probability for all classes with observation 781Numerical 0 probability for all classes with observation 325Numerical 0 probability for all classes with observation 344Numerical 0 probability for all classes with observation 807Numerical 0 probability for all classes with observation 358Numerical 0 probability for all classes with observation 359Numerical 0 probability for all classes with observation 826Numerical 0 probability for all classes with observation 846Numerical 0 probability for all classes with observation 848Numerical 0 probability for all classes with observation 409Numerical 0 probability for all classes with observation 852Numerical 0 probability for all classes with observation 855Numerical 0 probability for all classes with observation 426Numerical 0 probability for all classes with observation 862Numerical 0 probability for all classes with observation 882Numerical 0 probability for all classes with observation 888Numerical 0 probability for all classes with observation 892Numerical 0 probability for all classes with observation 893Numerical 0 probability for all classes with observation 479Numerical 0 probability for all classes with observation 489Numerical 0 probability for all classes with observation 492Numerical 0 probability for all classes with observation 494Numerical 0 probability for all classes with observation 500Numerical 0 probability for all classes with observation 926Numerical 0 probability for all classes with observation 936Numerical 0 probability for all classes with observation 937Numerical 0 probability for all classes with observation 531Numerical 0 probability for all classes with observation 948Numerical 0 probability for all classes with observation 538Numerical 0 probability for all classes with observation 542Numerical 0 probability for all classes with observation 956Numerical 0 probability for all classes with observation 957Numerical 0 probability for all classes with observation 973Numerical 0 probability for all classes with observation 571Numerical 0 probability for all classes with observation 975Numerical 0 probability for all classes with observation 588Numerical 0 probability for all classes with observation 991Numerical 0 probability for all classes with observation 1012Numerical 0 probability for all classes with observation 1018Numerical 0 probability for all classes with observation 606Numerical 0 probability for all classes with observation 625Numerical 0 probability for all classes with observation 630Numerical 0 probability for all classes with observation 662Numerical 0 probability for all classes with observation 678Numerical 0 probability for all classes with observation 684Numerical 0 probability for all classes with observation 686Numerical 0 probability for all classes with observation 6Numerical 0 probability for all classes with observation 704Numerical 0 probability for all classes with observation 21Numerical 0 probability for all classes with observation 723Numerical 0 probability for all classes with observation 38Numerical 0 probability for all classes with observation 42Numerical 0 probability for all classes with observation 742Numerical 0 probability for all classes with observation 744Numerical 0 probability for all classes with observation 78Numerical 0 probability for all classes with observation 79Numerical 0 probability for all classes with observation 794Numerical 0 probability for all classes with observation 800Numerical 0 probability for all classes with observation 96Numerical 0 probability for all classes with observation 824Numerical 0 probability for all classes with observation 116Numerical 0 probability for all classes with observation 118Numerical 0 probability for all classes with observation 119Numerical 0 probability for all classes with observation 834Numerical 0 probability for all classes with observation 843Numerical 0 probability for all classes with observation 129Numerical 0 probability for all classes with observation 850Numerical 0 probability for all classes with observation 859Numerical 0 probability for all classes with observation 142Numerical 0 probability for all classes with observation 152Numerical 0 probability for all classes with observation 882Numerical 0 probability for all classes with observation 159Numerical 0 probability for all classes with observation 885Numerical 0 probability for all classes with observation 191Numerical 0 probability for all classes with observation 922Numerical 0 probability for all classes with observation 194Numerical 0 probability for all classes with observation 927Numerical 0 probability for all classes with observation 932Numerical 0 probability for all classes with observation 938Numerical 0 probability for all classes with observation 944Numerical 0 probability for all classes with observation 948Numerical 0 probability for all classes with observation 218Numerical 0 probability for all classes with observation 234Numerical 0 probability for all classes with observation 235Numerical 0 probability for all classes with observation 977Numerical 0 probability for all classes with observation 984Numerical 0 probability for all classes with observation 993Numerical 0 probability for all classes with observation 997Numerical 0 probability for all classes with observation 261Numerical 0 probability for all classes with observation 1009Numerical 0 probability for all classes with observation 270Numerical 0 probability for all classes with observation 294Numerical 0 probability for all classes with observation 7Numerical 0 probability for all classes with observation 345Numerical 0 probability for all classes with observation 351Numerical 0 probability for all classes with observation 355Numerical 0 probability for all classes with observation 32Numerical 0 probability for all classes with observation 361Numerical 0 probability for all classes with observation 363Numerical 0 probability for all classes with observation 62Numerical 0 probability for all classes with observation 65Numerical 0 probability for all classes with observation 70Numerical 0 probability for all classes with observation 85Numerical 0 probability for all classes with observation 97Numerical 0 probability for all classes with observation 106Numerical 0 probability for all classes with observation 424Numerical 0 probability for all classes with observation 117Numerical 0 probability for all classes with observation 124Numerical 0 probability for all classes with observation 449Numerical 0 probability for all classes with observation 450Numerical 0 probability for all classes with observation 154Numerical 0 probability for all classes with observation 160Numerical 0 probability for all classes with observation 508Numerical 0 probability for all classes with observation 470Numerical 0 probability for all classes with observation 471Numerical 0 probability for all classes with observation 472Numerical 0 probability for all classes with observation 197Numerical 0 probability for all classes with observation 199Numerical 0 probability for all classes with observation 200Numerical 0 probability for all classes with observation 485Numerical 0 probability for all classes with observation 229Numerical 0 probability for all classes with observation 237Numerical 0 probability for all classes with observation 502Numerical 0 probability for all classes with observation 503Numerical 0 probability for all classes with observation 244Numerical 0 probability for all classes with observation 538Numerical 0 probability for all classes with observation 293Numerical 0 probability for all classes with observation 549Numerical 0 probability for all classes with observation 306Numerical 0 probability for all classes with observation 560Numerical 0 probability for all classes with observation 567Numerical 0 probability for all classes with observation 325Numerical 0 probability for all classes with observation 583Numerical 0 probability for all classes with observation 344Numerical 0 probability for all classes with observation 584Numerical 0 probability for all classes with observation 358Numerical 0 probability for all classes with observation 595Numerical 0 probability for all classes with observation 359Numerical 0 probability for all classes with observation 621Numerical 0 probability for all classes with observation 625Numerical 0 probability for all classes with observation 626Numerical 0 probability for all classes with observation 628Numerical 0 probability for all classes with observation 629Numerical 0 probability for all classes with observation 409Numerical 0 probability for all classes with observation 641Numerical 0 probability for all classes with observation 643Numerical 0 probability for all classes with observation 426Numerical 0 probability for all classes with observation 572Numerical 0 probability for all classes with observation 657Numerical 0 probability for all classes with observation 660Numerical 0 probability for all classes with observation 479Numerical 0 probability for all classes with observation 489Numerical 0 probability for all classes with observation 722Numerical 0 probability for all classes with observation 492Numerical 0 probability for all classes with observation 494Numerical 0 probability for all classes with observation 500Numerical 0 probability for all classes with observation 745Numerical 0 probability for all classes with observation 531Numerical 0 probability for all classes with observation 538Numerical 0 probability for all classes with observation 542Numerical 0 probability for all classes with observation 765Numerical 0 probability for all classes with observation 781Numerical 0 probability for all classes with observation 571Numerical 0 probability for all classes with observation 588Numerical 0 probability for all classes with observation 807Numerical 0 probability for all classes with observation 606Numerical 0 probability for all classes with observation 826Numerical 0 probability for all classes with observation 625Numerical 0 probability for all classes with observation 630Numerical 0 probability for all classes with observation 846Numerical 0 probability for all classes with observation 848Numerical 0 probability for all classes with observation 852Numerical 0 probability for all classes with observation 855Numerical 0 probability for all classes with observation 662Numerical 0 probability for all classes with observation 862Numerical 0 probability for all classes with observation 678Numerical 0 probability for all classes with observation 684Numerical 0 probability for all classes with observation 686Numerical 0 probability for all classes with observation 882Numerical 0 probability for all classes with observation 888Numerical 0 probability for all classes with observation 704Numerical 0 probability for all classes with observation 892Numerical 0 probability for all classes with observation 893Numerical 0 probability for all classes with observation 723Numerical 0 probability for all classes with observation 742Numerical 0 probability for all classes with observation 744Numerical 0 probability for all classes with observation 926Numerical 0 probability for all classes with observation 936Numerical 0 probability for all classes with observation 937Numerical 0 probability for all classes with observation 948Numerical 0 probability for all classes with observation 794Numerical 0 probability for all classes with observation 800Numerical 0 probability for all classes with observation 956Numerical 0 probability for all classes with observation 957Numerical 0 probability for all classes with observation 824Numerical 0 probability for all classes with observation 834Numerical 0 probability for all classes with observation 973Numerical 0 probability for all classes with observation 843Numerical 0 probability for all classes with observation 975Numerical 0 probability for all classes with observation 850Numerical 0 probability for all classes with observation 859Numerical 0 probability for all classes with observation 991Numerical 0 probability for all classes with observation 882Numerical 0 probability for all classes with observation 885Numerical 0 probability for all classes with observation 1012Numerical 0 probability for all classes with observation 1018Numerical 0 probability for all classes with observation 922Numerical 0 probability for all classes with observation 927Numerical 0 probability for all classes with observation 932Numerical 0 probability for all classes with observation 938Numerical 0 probability for all classes with observation 944Numerical 0 probability for all classes with observation 948Numerical 0 probability for all classes with observation 977Numerical 0 probability for all classes with observation 984Numerical 0 probability for all classes with observation 993Numerical 0 probability for all classes with observation 997Numerical 0 probability for all classes with observation 1009Numerical 0 probability for all classes with observation 14Numerical 0 probability for all classes with observation 20Numerical 0 probability for all classes with observation 35Numerical 0 probability for all classes with observation 49Numerical 0 probability for all classes with observation 51Numerical 0 probability for all classes with observation 52Numerical 0 probability for all classes with observation 7Numerical 0 probability for all classes with observation 16Numerical 0 probability for all classes with observation 107Numerical 0 probability for all classes with observation 71Numerical 0 probability for all classes with observation 85Numerical 0 probability for all classes with observation 163Numerical 0 probability for all classes with observation 99Numerical 0 probability for all classes with observation 185Numerical 0 probability for all classes with observation 155Numerical 0 probability for all classes with observation 163Numerical 0 probability for all classes with observation 164Numerical 0 probability for all classes with observation 239Numerical 0 probability for all classes with observation 192Numerical 0 probability for all classes with observation 264Numerical 0 probability for all classes with observation 215Numerical 0 probability for all classes with observation 218Numerical 0 probability for all classes with observation 219Numerical 0 probability for all classes with observation 239Numerical 0 probability for all classes with observation 242Numerical 0 probability for all classes with observation 296Numerical 0 probability for all classes with observation 322Numerical 0 probability for all classes with observation 283Numerical 0 probability for all classes with observation 315Numerical 0 probability for all classes with observation 331Numerical 0 probability for all classes with observation 342Numerical 0 probability for all classes with observation 361Numerical 0 probability for all classes with observation 343Numerical 0 probability for all classes with observation 345Numerical 0 probability for all classes with observation 371Numerical 0 probability for all classes with observation 380Numerical 0 probability for all classes with observation 381Numerical 0 probability for all classes with observation 382Numerical 0 probability for all classes with observation 365Numerical 0 probability for all classes with observation 384Numerical 0 probability for all classes with observation 388Numerical 0 probability for all classes with observation 397Numerical 0 probability for all classes with observation 414Numerical 0 probability for all classes with observation 416Numerical 0 probability for all classes with observation 420Numerical 0 probability for all classes with observation 427Numerical 0 probability for all classes with observation 254Numerical 0 probability for all classes with observation 446Numerical 0 probability for all classes with observation 449Numerical 0 probability for all classes with observation 471Numerical 0 probability for all classes with observation 497Numerical 0 probability for all classes with observation 511Numerical 0 probability for all classes with observation 512Numerical 0 probability for all classes with observation 533Numerical 0 probability for all classes with observation 529Numerical 0 probability for all classes with observation 534Numerical 0 probability for all classes with observation 566Numerical 0 probability for all classes with observation 567Numerical 0 probability for all classes with observation 571Numerical 0 probability for all classes with observation 577Numerical 0 probability for all classes with observation 568Numerical 0 probability for all classes with observation 602Numerical 0 probability for all classes with observation 613Numerical 0 probability for all classes with observation 580Numerical 0 probability for all classes with observation 621Numerical 0 probability for all classes with observation 640Numerical 0 probability for all classes with observation 649Numerical 0 probability for all classes with observation 650Numerical 0 probability for all classes with observation 666Numerical 0 probability for all classes with observation 634Numerical 0 probability for all classes with observation 637Numerical 0 probability for all classes with observation 649Numerical 0 probability for all classes with observation 700Numerical 0 probability for all classes with observation 701Numerical 0 probability for all classes with observation 664Numerical 0 probability for all classes with observation 719Numerical 0 probability for all classes with observation 720Numerical 0 probability for all classes with observation 723Numerical 0 probability for all classes with observation 726Numerical 0 probability for all classes with observation 727Numerical 0 probability for all classes with observation 740Numerical 0 probability for all classes with observation 741Numerical 0 probability for all classes with observation 708Numerical 0 probability for all classes with observation 764Numerical 0 probability for all classes with observation 719Numerical 0 probability for all classes with observation 777Numerical 0 probability for all classes with observation 726Numerical 0 probability for all classes with observation 727Numerical 0 probability for all classes with observation 739Numerical 0 probability for all classes with observation 826Numerical 0 probability for all classes with observation 759Numerical 0 probability for all classes with observation 836Numerical 0 probability for all classes with observation 772Numerical 0 probability for all classes with observation 774Numerical 0 probability for all classes with observation 853Numerical 0 probability for all classes with observation 780Numerical 0 probability for all classes with observation 864Numerical 0 probability for all classes with observation 869Numerical 0 probability for all classes with observation 877Numerical 0 probability for all classes with observation 801Numerical 0 probability for all classes with observation 888Numerical 0 probability for all classes with observation 812Numerical 0 probability for all classes with observation 389Numerical 0 probability for all classes with observation 833Numerical 0 probability for all classes with observation 904Numerical 0 probability for all classes with observation 847Numerical 0 probability for all classes with observation 917Numerical 0 probability for all classes with observation 851Numerical 0 probability for all classes with observation 923Numerical 0 probability for all classes with observation 860Numerical 0 probability for all classes with observation 936Numerical 0 probability for all classes with observation 867Numerical 0 probability for all classes with observation 954Numerical 0 probability for all classes with observation 884Numerical 0 probability for all classes with observation 890Numerical 0 probability for all classes with observation 899Numerical 0 probability for all classes with observation 993Numerical 0 probability for all classes with observation 1000Numerical 0 probability for all classes with observation 1006Numerical 0 probability for all classes with observation 909Numerical 0 probability for all classes with observation 917Numerical 0 probability for all classes with observation 920Numerical 0 probability for all classes with observation 935Numerical 0 probability for all classes with observation 941Numerical 0 probability for all classes with observation 7Numerical 0 probability for all classes with observation 16Numerical 0 probability for all classes with observation 957Numerical 0 probability for all classes with observation 71Numerical 0 probability for all classes with observation 998Numerical 0 probability for all classes with observation 85Numerical 0 probability for all classes with observation 1004Numerical 0 probability for all classes with observation 1005Numerical 0 probability for all classes with observation 1011Numerical 0 probability for all classes with observation 99Numerical 0 probability for all classes with observation 1021Numerical 0 probability for all classes with observation 155Numerical 0 probability for all classes with observation 163Numerical 0 probability for all classes with observation 164Numerical 0 probability for all classes with observation 14Numerical 0 probability for all classes with observation 20Numerical 0 probability for all classes with observation 192Numerical 0 probability for all classes with observation 35Numerical 0 probability for all classes with observation 215Numerical 0 probability for all classes with observation 218Numerical 0 probability for all classes with observation 219Numerical 0 probability for all classes with observation 49Numerical 0 probability for all classes with observation 51Numerical 0 probability for all classes with observation 52Numerical 0 probability for all classes with observation 239Numerical 0 probability for all classes with observation 242Numerical 0 probability for all classes with observation 283Numerical 0 probability for all classes with observation 107Numerical 0 probability for all classes with observation 315Numerical 0 probability for all classes with observation 331Numerical 0 probability for all classes with observation 343Numerical 0 probability for all classes with observation 345Numerical 0 probability for all classes with observation 163Numerical 0 probability for all classes with observation 365Numerical 0 probability for all classes with observation 185Numerical 0 probability for all classes with observation 388Numerical 0 probability for all classes with observation 397Numerical 0 probability for all classes with observation 239Numerical 0 probability for all classes with observation 446Numerical 0 probability for all classes with observation 264Numerical 0 probability for all classes with observation 296Numerical 0 probability for all classes with observation 497Numerical 0 probability for all classes with observation 322Numerical 0 probability for all classes with observation 533Numerical 0 probability for all classes with observation 342Numerical 0 probability for all classes with observation 566Numerical 0 probability for all classes with observation 567Numerical 0 probability for all classes with observation 361Numerical 0 probability for all classes with observation 571Numerical 0 probability for all classes with observation 577Numerical 0 probability for all classes with observation 371Numerical 0 probability for all classes with observation 380Numerical 0 probability for all classes with observation 381Numerical 0 probability for all classes with observation 382Numerical 0 probability for all classes with observation 602Numerical 0 probability for all classes with observation 384Numerical 0 probability for all classes with observation 613Numerical 0 probability for all classes with observation 621Numerical 0 probability for all classes with observation 640Numerical 0 probability for all classes with observation 649Numerical 0 probability for all classes with observation 650Numerical 0 probability for all classes with observation 666Numerical 0 probability for all classes with observation 414Numerical 0 probability for all classes with observation 416Numerical 0 probability for all classes with observation 420Numerical 0 probability for all classes with observation 427Numerical 0 probability for all classes with observation 700Numerical 0 probability for all classes with observation 701Numerical 0 probability for all classes with observation 449Numerical 0 probability for all classes with observation 719Numerical 0 probability for all classes with observation 720Numerical 0 probability for all classes with observation 723Numerical 0 probability for all classes with observation 726Numerical 0 probability for all classes with observation 727Numerical 0 probability for all classes with observation 740Numerical 0 probability for all classes with observation 471Numerical 0 probability for all classes with observation 741Numerical 0 probability for all classes with observation 764Numerical 0 probability for all classes with observation 777Numerical 0 probability for all classes with observation 511Numerical 0 probability for all classes with observation 512Numerical 0 probability for all classes with observation 529Numerical 0 probability for all classes with observation 534Numerical 0 probability for all classes with observation 826Numerical 0 probability for all classes with observation 836Numerical 0 probability for all classes with observation 853Numerical 0 probability for all classes with observation 568Numerical 0 probability for all classes with observation 864Numerical 0 probability for all classes with observation 869Numerical 0 probability for all classes with observation 580Numerical 0 probability for all classes with observation 877Numerical 0 probability for all classes with observation 888Numerical 0 probability for all classes with observation 904Numerical 0 probability for all classes with observation 917Numerical 0 probability for all classes with observation 923Numerical 0 probability for all classes with observation 936Numerical 0 probability for all classes with observation 634Numerical 0 probability for all classes with observation 637Numerical 0 probability for all classes with observation 954Numerical 0 probability for all classes with observation 649Numerical 0 probability for all classes with observation 664Numerical 0 probability for all classes with observation 993Numerical 0 probability for all classes with observation 708Numerical 0 probability for all classes with observation 1000Numerical 0 probability for all classes with observation 1006Numerical 0 probability for all classes with observation 719Numerical 0 probability for all classes with observation 726Numerical 0 probability for all classes with observation 727Numerical 0 probability for all classes with observation 739Numerical 0 probability for all classes with observation 759Numerical 0 probability for all classes with observation 772Numerical 0 probability for all classes with observation 774Numerical 0 probability for all classes with observation 780Numerical 0 probability for all classes with observation 801Numerical 0 probability for all classes with observation 812Numerical 0 probability for all classes with observation 833Numerical 0 probability for all classes with observation 847Numerical 0 probability for all classes with observation 851Numerical 0 probability for all classes with observation 860Numerical 0 probability for all classes with observation 867Numerical 0 probability for all classes with observation 884Numerical 0 probability for all classes with observation 890Numerical 0 probability for all classes with observation 899Numerical 0 probability for all classes with observation 39Numerical 0 probability for all classes with observation 47Numerical 0 probability for all classes with observation 909Numerical 0 probability for all classes with observation 59Numerical 0 probability for all classes with observation 917Numerical 0 probability for all classes with observation 920Numerical 0 probability for all classes with observation 78Numerical 0 probability for all classes with observation 935Numerical 0 probability for all classes with observation 88Numerical 0 probability for all classes with observation 941Numerical 0 probability for all classes with observation 118Numerical 0 probability for all classes with observation 957Numerical 0 probability for all classes with observation 153Numerical 0 probability for all classes with observation 173Numerical 0 probability for all classes with observation 998Numerical 0 probability for all classes with observation 1004Numerical 0 probability for all classes with observation 1005Numerical 0 probability for all classes with observation 1011Numerical 0 probability for all classes with observation 1021Numerical 0 probability for all classes with observation 264Numerical 0 probability for all classes with observation 265Numerical 0 probability for all classes with observation 273Numerical 0 probability for all classes with observation 290Numerical 0 probability for all classes with observation 301Numerical 0 probability for all classes with observation 309Numerical 0 probability for all classes with observation 321Numerical 0 probability for all classes with observation 332Numerical 0 probability for all classes with observation 369Numerical 0 probability for all classes with observation 406Numerical 0 probability for all classes with observation 21Numerical 0 probability for all classes with observation 23Numerical 0 probability for all classes with observation 472Numerical 0 probability for all classes with observation 927Numerical 0 probability for all classes with observation 490Numerical 0 probability for all classes with observation 50Numerical 0 probability for all classes with observation 67Numerical 0 probability for all classes with observation 533Numerical 0 probability for all classes with observation 544Numerical 0 probability for all classes with observation 97Numerical 0 probability for all classes with observation 555Numerical 0 probability for all classes with observation 102Numerical 0 probability for all classes with observation 608Numerical 0 probability for all classes with observation 615Numerical 0 probability for all classes with observation 153Numerical 0 probability for all classes with observation 624Numerical 0 probability for all classes with observation 625Numerical 0 probability for all classes with observation 626Numerical 0 probability for all classes with observation 641Numerical 0 probability for all classes with observation 176Numerical 0 probability for all classes with observation 645Numerical 0 probability for all classes with observation 177Numerical 0 probability for all classes with observation 650Numerical 0 probability for all classes with observation 665Numerical 0 probability for all classes with observation 192Numerical 0 probability for all classes with observation 666Numerical 0 probability for all classes with observation 668Numerical 0 probability for all classes with observation 670Numerical 0 probability for all classes with observation 201Numerical 0 probability for all classes with observation 208Numerical 0 probability for all classes with observation 693Numerical 0 probability for all classes with observation 215Numerical 0 probability for all classes with observation 694Numerical 0 probability for all classes with observation 219Numerical 0 probability for all classes with observation 698Numerical 0 probability for all classes with observation 708Numerical 0 probability for all classes with observation 361Numerical 0 probability for all classes with observation 737Numerical 0 probability for all classes with observation 739Numerical 0 probability for all classes with observation 268Numerical 0 probability for all classes with observation 284Numerical 0 probability for all classes with observation 293Numerical 0 probability for all classes with observation 792Numerical 0 probability for all classes with observation 802Numerical 0 probability for all classes with observation 804Numerical 0 probability for all classes with observation 806Numerical 0 probability for all classes with observation 817Numerical 0 probability for all classes with observation 819Numerical 0 probability for all classes with observation 320Numerical 0 probability for all classes with observation 832Numerical 0 probability for all classes with observation 837Numerical 0 probability for all classes with observation 340Numerical 0 probability for all classes with observation 884Numerical 0 probability for all classes with observation 382Numerical 0 probability for all classes with observation 900Numerical 0 probability for all classes with observation 909Numerical 0 probability for all classes with observation 398Numerical 0 probability for all classes with observation 916Numerical 0 probability for all classes with observation 921Numerical 0 probability for all classes with observation 419Numerical 0 probability for all classes with observation 945Numerical 0 probability for all classes with observation 983Numerical 0 probability for all classes with observation 448Numerical 0 probability for all classes with observation 454Numerical 0 probability for all classes with observation 1004Numerical 0 probability for all classes with observation 1014Numerical 0 probability for all classes with observation 498Numerical 0 probability for all classes with observation 512Numerical 0 probability for all classes with observation 518Numerical 0 probability for all classes with observation 532Numerical 0 probability for all classes with observation 537Numerical 0 probability for all classes with observation 539Numerical 0 probability for all classes with observation 39Numerical 0 probability for all classes with observation 546Numerical 0 probability for all classes with observation 47Numerical 0 probability for all classes with observation 59Numerical 0 probability for all classes with observation 561Numerical 0 probability for all classes with observation 78Numerical 0 probability for all classes with observation 88Numerical 0 probability for all classes with observation 118Numerical 0 probability for all classes with observation 153Numerical 0 probability for all classes with observation 641Numerical 0 probability for all classes with observation 173Numerical 0 probability for all classes with observation 671Numerical 0 probability for all classes with observation 675Numerical 0 probability for all classes with observation 712Numerical 0 probability for all classes with observation 713Numerical 0 probability for all classes with observation 264Numerical 0 probability for all classes with observation 265Numerical 0 probability for all classes with observation 725Numerical 0 probability for all classes with observation 273Numerical 0 probability for all classes with observation 744Numerical 0 probability for all classes with observation 290Numerical 0 probability for all classes with observation 301Numerical 0 probability for all classes with observation 309Numerical 0 probability for all classes with observation 763Numerical 0 probability for all classes with observation 770Numerical 0 probability for all classes with observation 321Numerical 0 probability for all classes with observation 777Numerical 0 probability for all classes with observation 332Numerical 0 probability for all classes with observation 790Numerical 0 probability for all classes with observation 800Numerical 0 probability for all classes with observation 369Numerical 0 probability for all classes with observation 822Numerical 0 probability for all classes with observation 828Numerical 0 probability for all classes with observation 833Numerical 0 probability for all classes with observation 834Numerical 0 probability for all classes with observation 406Numerical 0 probability for all classes with observation 837Numerical 0 probability for all classes with observation 850Numerical 0 probability for all classes with observation 869Numerical 0 probability for all classes with observation 883Numerical 0 probability for all classes with observation 472Numerical 0 probability for all classes with observation 887Numerical 0 probability for all classes with observation 903Numerical 0 probability for all classes with observation 490Numerical 0 probability for all classes with observation 533Numerical 0 probability for all classes with observation 965Numerical 0 probability for all classes with observation 967Numerical 0 probability for all classes with observation 544Numerical 0 probability for all classes with observation 555Numerical 0 probability for all classes with observation 989Numerical 0 probability for all classes with observation 991Numerical 0 probability for all classes with observation 608Numerical 0 probability for all classes with observation 615Numerical 0 probability for all classes with observation 624Numerical 0 probability for all classes with observation 625Numerical 0 probability for all classes with observation 626Numerical 0 probability for all classes with observation 641Numerical 0 probability for all classes with observation 645Numerical 0 probability for all classes with observation 650Numerical 0 probability for all classes with observation 254Numerical 0 probability for all classes with observation 665Numerical 0 probability for all classes with observation 666Numerical 0 probability for all classes with observation 668Numerical 0 probability for all classes with observation 670Numerical 0 probability for all classes with observation 693Numerical 0 probability for all classes with observation 694Numerical 0 probability for all classes with observation 698Numerical 0 probability for all classes with observation 708Numerical 0 probability for all classes with observation 21Numerical 0 probability for all classes with observation 23Numerical 0 probability for all classes with observation 737Numerical 0 probability for all classes with observation 739Numerical 0 probability for all classes with observation 50Numerical 0 probability for all classes with observation 67Numerical 0 probability for all classes with observation 792Numerical 0 probability for all classes with observation 802Numerical 0 probability for all classes with observation 97Numerical 0 probability for all classes with observation 804Numerical 0 probability for all classes with observation 806Numerical 0 probability for all classes with observation 102Numerical 0 probability for all classes with observation 817Numerical 0 probability for all classes with observation 819Numerical 0 probability for all classes with observation 832Numerical 0 probability for all classes with observation 837Numerical 0 probability for all classes with observation 153Numerical 0 probability for all classes with observation 884Numerical 0 probability for all classes with observation 176Numerical 0 probability for all classes with observation 900Numerical 0 probability for all classes with observation 177Numerical 0 probability for all classes with observation 909Numerical 0 probability for all classes with observation 916Numerical 0 probability for all classes with observation 192Numerical 0 probability for all classes with observation 921Numerical 0 probability for all classes with observation 201Numerical 0 probability for all classes with observation 208Numerical 0 probability for all classes with observation 945Numerical 0 probability for all classes with observation 215Numerical 0 probability for all classes with observation 219Numerical 0 probability for all classes with observation 983Numerical 0 probability for all classes with observation 1004Numerical 0 probability for all classes with observation 268Numerical 0 probability for all classes with observation 1014Numerical 0 probability for all classes with observation 284Numerical 0 probability for all classes with observation 293Numerical 0 probability for all classes with observation 320Numerical 0 probability for all classes with observation 340Numerical 0 probability for all classes with observation 389Numerical 0 probability for all classes with observation 382Numerical 0 probability for all classes with observation 398Numerical 0 probability for all classes with observation 419Numerical 0 probability for all classes with observation 448Numerical 0 probability for all classes with observation 454Numerical 0 probability for all classes with observation 498Numerical 0 probability for all classes with observation 512Numerical 0 probability for all classes with observation 518Numerical 0 probability for all classes with observation 532Numerical 0 probability for all classes with observation 537Numerical 0 probability for all classes with observation 539Numerical 0 probability for all classes with observation 546Numerical 0 probability for all classes with observation 561Numerical 0 probability for all classes with observation 641Numerical 0 probability for all classes with observation 671Numerical 0 probability for all classes with observation 675Numerical 0 probability for all classes with observation 712Numerical 0 probability for all classes with observation 713Numerical 0 probability for all classes with observation 725Numerical 0 probability for all classes with observation 744Numerical 0 probability for all classes with observation 763Numerical 0 probability for all classes with observation 770Numerical 0 probability for all classes with observation 777Numerical 0 probability for all classes with observation 790Numerical 0 probability for all classes with observation 800Numerical 0 probability for all classes with observation 822Numerical 0 probability for all classes with observation 828Numerical 0 probability for all classes with observation 833Numerical 0 probability for all classes with observation 834Numerical 0 probability for all classes with observation 837Numerical 0 probability for all classes with observation 850Numerical 0 probability for all classes with observation 869Numerical 0 probability for all classes with observation 883Numerical 0 probability for all classes with observation 887Numerical 0 probability for all classes with observation 903Numerical 0 probability for all classes with observation 965Numerical 0 probability for all classes with observation 967Numerical 0 probability for all classes with observation 989Numerical 0 probability for all classes with observation 991Numerical 0 probability for all classes with observation 955Numerical 0 probability for all classes with observation 927Numerical 0 probability for all classes with observation 361Numerical 0 probability for all classes with observation 196Numerical 0 probability for all classes with observation 955Numerical 0 probability for all classes with observation 744Numerical 0 probability for all classes with observation 993Numerical 0 probability for all classes with observation 472Numerical 0 probability for all classes with observation 196Numerical 0 probability for all classes with observation 744Numerical 0 probability for all classes with observation 993Numerical 0 probability for all classes with observation 118Numerical 0 probability for all classes with observation 472Numerical 0 probability for all classes with observation 50Numerical 0 probability for all classes with observation 325Numerical 0 probability for all classes with observation 118Numerical 0 probability for all classes with observation 50Numerical 0 probability for all classes with observation 325

Aggregating results
Selecting tuning parameters
Fitting fL = 0, usekernel = TRUE, adjust = 1 on full training set

naiveBayesFit

Naive Bayes 

10256 samples
   10 predictors
    2 classes: 'Botnet', 'Normal' 

Pre-processing: scaled (10), centered (10) 
Resampling: Cross-Validated (10 fold) 
Summary of sample sizes: 9230, 9230, 9230, 9230, 9230, 9231, ... 
Resampling results across tuning parameters:

  usekernel  ROC        Sens       Spec     
  FALSE      0.9287258  0.8307368  0.8863072
   TRUE      0.9427740  0.7794503  0.9465700

Tuning parameter 'fL' was held constant at a value of 0
Tuning parameter 'adjust' was held constant at a value of 1
ROC was used to select the optimal model using  the largest value.
The final values used for the model were fL = 0, usekernel = TRUE and adjust = 1. 

naiveBayesPredict <- predict(naiveBayesFit, newdata = testing)

Numerical 0 probability for all classes with observation 96

confusionMatrix(naiveBayesPredict, testing$class)

Confusion Matrix and Statistics

          Reference
Prediction Botnet Normal
    Botnet   1022     39
    Normal    260    530
                                         
               Accuracy : 0.8385         
                 95% CI : (0.8209, 0.855)
    No Information Rate : 0.6926         
    P-Value [Acc > NIR] : < 2.2e-16      
                                         
                  Kappa : 0.6576         
 Mcnemar's Test P-Value : < 2.2e-16      
                                         
            Sensitivity : 0.7972         
            Specificity : 0.9315         
         Pos Pred Value : 0.9632         
         Neg Pred Value : 0.6709         
             Prevalence : 0.6926         
         Detection Rate : 0.5521         
   Detection Prevalence : 0.5732         
      Balanced Accuracy : 0.8643         
                                         
       'Positive' Class : Botnet         
                                         

naiveBayesPredict <- predict(naiveBayesFit, newdata = testing, type = 'prob')

Numerical 0 probability for all classes with observation 96

naiveBayesROC <- roc(testing$class,naiveBayesPredict[,"Botnet"], levels = c('Normal','Botnet'))#rev(testing$class))
naiveBayesROC

Call:
roc.default(response = testing$class, predictor = naiveBayesPredict[,     "Botnet"], levels = c("Normal", "Botnet"))

Data: naiveBayesPredict[, "Botnet"] in 569 controls (testing$class Normal) < 1282 cases (testing$class Botnet).
Area under the curve: 0.9343

ggplot(cbind(naiveBayesPredict,class=testing$class), 
       aes(m = Botnet, d = factor(class, labels=c("Normal","Botnet"),levels = c("Normal", "Botnet")))) + 
    geom_roc(hjust = -0.4, vjust = 1.5,colour='orange') + 
  theme_bw()

#plot(naiveBayesROC, type="S", print.thres= 0.5)

Suport Vector Machine

svmFit <- train(class ~ sp+wp+wnp+snp+ds+dm+dl+ss+sm+sl, method='svmLinear', trControl = ctrl_fast,preProcess=c('scale', 'center'), data=training, family=binomial(link='logit'))

The metric "Accuracy" was not in the result set. ROC will be used instead.kernlab class prediction calculations failed; returning NAskernlab class probability calculations failed; returning NAsrow names were found from a short variable and have been discardedkernlab class prediction calculations failed; returning NAskernlab class probability calculations failed; returning NAsrow names were found from a short variable and have been discardedkernlab class prediction calculations failed; returning NAskernlab class probability calculations failed; returning NAsrow names were found from a short variable and have been discardedkernlab class prediction calculations failed; returning NAskernlab class probability calculations failed; returning NAsrow names were found from a short variable and have been discardedkernlab class prediction calculations failed; returning NAskernlab class probability calculations failed; returning NAsrow names were found from a short variable and have been discardedThere were missing values in resampled performance measures.

Aggregating results
Fitting final model on full training set

svmFit

Support Vector Machines with Linear Kernel 

10256 samples
   10 predictors
    2 classes: 'Botnet', 'Normal' 

Pre-processing: scaled (10), centered (10) 
Resampling: Cross-Validated (10 fold) 
Summary of sample sizes: 9230, 9230, 9231, 9231, 9231, 9230, ... 
Resampling results:

  ROC        Sens       Spec     
  0.5920493  0.8389346  0.9063893

Tuning parameter 'C' was held constant at a value of 1
 

svmPredict <- predict(svmFit, newdata = testing)
confusionMatrix(svmPredict, testing$class)

Confusion Matrix and Statistics

          Reference
Prediction Botnet Normal
    Botnet   1087     51
    Normal    195    518
                                          
               Accuracy : 0.8671          
                 95% CI : (0.8508, 0.8822)
    No Information Rate : 0.6926          
    P-Value [Acc > NIR] : < 2.2e-16       
                                          
                  Kappa : 0.7084          
 Mcnemar's Test P-Value : < 2.2e-16       
                                          
            Sensitivity : 0.8479          
            Specificity : 0.9104          
         Pos Pred Value : 0.9552          
         Neg Pred Value : 0.7265          
             Prevalence : 0.6926          
         Detection Rate : 0.5873          
   Detection Prevalence : 0.6148          
      Balanced Accuracy : 0.8791          
                                          
       'Positive' Class : Botnet          
                                          

svmPredict <- predict(svmFit, newdata = testing, type = "prob")
svmROC <- roc(testing$class,svmPredict[,"Botnet"], levels = c('Normal','Botnet'))#rev(testing$class))
svmROC

Call:
roc.default(response = testing$class, predictor = svmPredict[,     "Botnet"], levels = c("Normal", "Botnet"))

Data: svmPredict[, "Botnet"] in 569 controls (testing$class Normal) < 1282 cases (testing$class Botnet).
Area under the curve: 0.9373

ggplot(cbind(svmPredict,class=testing$class), 
       aes(m = Botnet, d = factor(class, labels=c("Normal","Botnet"),levels = c("Normal", "Botnet")))) + 
    geom_roc(hjust = -0.4, vjust = 1.5,colour='orange') + 
  theme_bw()

Comparing Models

resamps <- resamples(list(rf = rfFit, lr = logicRFit, nv = naiveBayesFit, svm = svmFit))
summary(resamps)

Call:
summary.resamples(object = resamps)

Models: rf, lr, nv, svm 
Number of resamples: 10 

ROC 
      Min. 1st Qu. Median   Mean 3rd Qu.   Max. NA's
rf  0.9692  0.9727 0.9766 0.9761  0.9786 0.9849    0
lr  0.9299  0.9364 0.9399 0.9402  0.9448 0.9477    0
nv  0.9372  0.9397 0.9426 0.9428  0.9442 0.9534    0
svm 0.2500  0.2500 0.5825 0.5920  0.9363 0.9497    0

Sens 
      Min. 1st Qu. Median   Mean 3rd Qu.   Max. NA's
rf  0.8986  0.9142 0.9162 0.9204  0.9253 0.9474    0
lr  0.7992  0.8328 0.8421 0.8374  0.8470 0.8613    0
nv  0.7661  0.7719 0.7785 0.7795  0.7836 0.7988    0
svm 0.8031  0.8187 0.8382 0.8389  0.8672 0.8674    5

Spec 
      Min. 1st Qu. Median   Mean 3rd Qu.   Max. NA's
rf  0.9571  0.9629 0.9649 0.9680  0.9722 0.9844    0
lr  0.8967  0.9069 0.9171 0.9167  0.9254 0.9376    0
nv  0.9279  0.9381 0.9483 0.9466  0.9551 0.9669    0
svm 0.8887  0.9045 0.9103 0.9064  0.9123 0.9162    5

bwplot(resamps)

diffs <- diff(resamps)
summary(diffs)

Call:
summary.diff.resamples(object = diffs)

p-value adjustment: bonferroni 
Upper diagonal: estimates of the difference
Lower diagonal: p-value for H0: difference = 0

ROC 
    rf        lr        nv        svm      
rf             0.035901  0.033320  0.384045
lr  1.615e-06           -0.002581  0.348144
nv  1.883e-07 1.00000              0.350725
svm 0.04715   0.08462   0.07869            

Sens 
    rf        lr        nv        svm      
rf             0.083072  0.140986  0.090471
lr  8.132e-06            0.057915 -0.006646
nv  5.034e-08 2.202e-05           -0.064361
svm 0.01160   1.00000   0.05475            

Spec 
    rf        lr        nv        svm      
rf             0.051283  0.021447  0.062422
lr  1.321e-05           -0.029836  0.008198
nv  0.0007621 0.0070079            0.045247
svm 0.0097445 1.0000000 0.0076966          

values=resamps$values
values

names(values)[2]<-"rfSens"
ggplot(values)+
  geom_boxplot(aes(y=rfSens,x=1))

Making probabilistic table.

# Botnet probabilistic table
botnet_prob_result = data.frame(testing$class, predsrfprobs$Botnet, knnPredict$Botnet, logicRPredict$Botnet, naiveBayesPredict$Botnet ,svmPredict$Botnet, testing$subclass, testing$port, testing$proto)
#botnet_prob_result = botnet_prob_result %>% mutate(subclass = data_test$subclass)
names(botnet_prob_result) = c('TrueClass','RamdomForest','KNN','LogisticRegression', 'NaiveBayes', 'SVM','subclass','port','proto')
botnet_prob_result

#load("./botnet_prob_results.Rda")
library(grid)
library(gridExtra)
botnet_prob_result %>% group_by(subclass) %>% summarise(n=n(),sum_RF=sum(RamdomForest),sum_KNN=sum(KNN),sum_LR=sum(LogisticRegression),sum_NB=sum(NaiveBayes),sum_SVM=sum(SVM)) %>% arrange(desc(n))

botnet_prob_result %>% group_by(subclass) %>% summarise(n=n(),mean=mean(NaiveBayes),sd=sd(NaiveBayes)) %>% arrange(desc(n))%>% top_n(10)

Selecting by sd

botnet_10_top<-botnet_prob_result %>% filter(TrueClass == 'Botnet') %>% group_by(subclass) %>% summarise(n=n()) %>% arrange(desc(n))%>% top_n(10)

Selecting by n

botnet_10_top<-inner_join(botnet_10_top,botnet_prob_result,by="subclass")
rf_plot<-bwplot(subclass~RamdomForest,data=botnet_10_top,do.out = FALSE,scales=list(y=list(draw=FALSE)))
knn_plot<-bwplot(subclass~KNN,data=botnet_10_top,do.out = FALSE,scales=list(y=list(draw=FALSE)))
rl_plot<-bwplot(subclass~LogisticRegression,data=botnet_10_top,do.out = FALSE,scales=list(y=list(draw=FALSE)))
svm_plot<-bwplot(subclass~SVM,data=botnet_10_top,do.out = FALSE,scales=list(y=list(draw=FALSE)))
pl = list(knn_plot, rf_plot,rl_plot,svm_plot)
# do.call(grid.arrange, c(pl, nrow=1))
do.call(grid.arrange, c(lapply(pl, update), list(nrow=1)))

some heatmaps

library(scales)
knn_m<-matrix(botnet_prob_result[1:1830,]$KNN,ncol=30,nrow=61)
svm_m<-matrix(botnet_prob_result[1:1830,]$SVM,ncol=30,nrow=61)
lr_m<-matrix(botnet_prob_result[1:1830,]$LogisticRegression,ncol=30,nrow=61)
nb_m<-matrix(botnet_prob_result[1:1830,]$NaiveBayes,ncol=30,nrow=61)
rf_m<-matrix(botnet_prob_result[1:1830,]$RamdomForest,ncol=30,nrow=61)
mdf<-as.data.frame(knn_m)
mdf<-cbind(mdf,id=seq(1:61))
mdf<-reshape2::melt(mdf,id.vars=c("id"))
h1<-ggplot(mdf)+
  geom_tile(aes(x=id,y=variable,fill=value),
            colour = "white") +
  scale_fill_gradient(low = "white",
    high = "orange")+ylab("")+xlab("")+
  guides(fill=FALSE)
mdf<-as.data.frame(svm_m)
mdf<-cbind(mdf,id=seq(1:61))
mdf<-reshape2::melt(mdf,id.vars=c("id"))
h2<-ggplot(mdf)+
  geom_tile(aes(x=id,y=variable,fill=value),
            colour = "white") +
  scale_fill_gradient(low = "white",
    high = "orange")+ylab("")+xlab("")+
  guides(fill=FALSE)
mdf<-as.data.frame(rf_m)
mdf<-cbind(mdf,id=seq(1:61))
mdf<-reshape2::melt(mdf,id.vars=c("id"))
h3<-ggplot(mdf)+
  geom_tile(aes(x=id,y=variable,fill=value),
            colour = "white") +
  scale_fill_gradient(low = "white",
    high = "orange")+ylab("")+xlab("")+
  guides(fill=FALSE)
mdf<-as.data.frame(nb_m)
mdf<-cbind(mdf,id=seq(1:61))
mdf<-reshape2::melt(mdf,id.vars=c("id"))
h4<-ggplot(mdf)+
  geom_tile(aes(x=id,y=variable,fill=value),
            colour = "white") +
  scale_fill_gradient(low = "white",
    high = "orange")+ylab("")+xlab("")+
  guides(fill=FALSE)
grid.arrange(h1,h2,h3,h4,ncol=2,nrow=2)

difference heatmaps

mdf<-as.data.frame(rf_m - knn_m)
mdf<-cbind(mdf,id=seq(1:61))
mdf<-reshape2::melt(mdf,id.vars=c("id"))
mdf<-cbind(mdf,subclass=(botnet_prob_result[1:1830,]$subclass))
diff<-ggplot(mdf)+
  geom_tile(aes(x=id,y=variable,fill=value,text=subclass),
            colour = "white") +
  scale_fill_gradientn(colours=c("red","white","green"),
           values  = rescale(c(min(mdf$value), 0.05, max(mdf$value))))+
           guides(fill=FALSE)+theme_bw()  

Ignoring unknown aesthetics: text

ggplotly(diff)

We recommend that you use the dev version of ggplot2 with `ggplotly()`
Install it with: `devtools::install_github('hadley/ggplot2')`

d3heatmap(rf_m - knn_m,colors = "Blues",cellnote=matrix(botnet_prob_result[1:1830,]$subclass,ncol=30,nrow=61))

Subclass probability analisys (Attempt I)

KNN vs RF.

a=mdf %>% filter(value< -0.09) %>% group_by(subclass) %>% summarise(totless009=n()) %>% arrange(desc(totless009))
b=mdf  %>% group_by(subclass) %>% summarise(total=n()) %>% arrange(desc(total))
subclass_percent_diff<-inner_join(a,b,by="subclass") %>% mutate(percent=totless009/total) %>% arrange(desc(total)) 
subclass_percent_diff

subclass_detections<-botnet_prob_result %>% mutate(detected_rf=ifelse(RamdomForest>0.5,"Botnet","Normal"),
                              detected_knn=ifelse(KNN>0.5,"Botnet","Normal")) %>% 
                              mutate(correct_rf=ifelse(detected_rf==TrueClass,1,0))%>%
                              mutate(correct_knn=ifelse(detected_knn==TrueClass,1,0)) %>% 
  group_by(subclass) %>% summarise(total_correct_rf=sum(correct_rf),total_correct_knn=sum(correct_knn))
inner_join(subclass_detections,subclass_percent_diff,by="subclass")

mdf<-cbind(mdf,rf=botnet_prob_result$RamdomForest[1:1830],knn=botnet_prob_result$KNN[1:1830],trueclass=botnet_prob_result$TrueClass[1:1830])
botnet_prob_result

mdf %>% filter(value< -0.09) %>% filter(trueclass=="Botnet")

mdf %>% filter(value> 0.09) %>% filter(trueclass=="Normal")

Useful functions

multiplot <- function(..., plotlist=NULL, file, cols=1, layout=NULL) {
  library(grid)
  # Make a list from the ... arguments and plotlist
  plots <- c(list(...), plotlist)
  numPlots = length(plots)
  # If layout is NULL, then use 'cols' to determine layout
  if (is.null(layout)) {
    # Make the panel
    # ncol: Number of columns of plots
    # nrow: Number of rows needed, calculated from # of cols
    layout <- matrix(seq(1, cols * ceiling(numPlots/cols)),
                    ncol = cols, nrow = ceiling(numPlots/cols))
  }
 if (numPlots==1) {
    print(plots[[1]])
  } else {
    # Set up the page
    grid.newpage()
    pushViewport(viewport(layout = grid.layout(nrow(layout), ncol(layout))))
    # Make each plot, in the correct location
    for (i in 1:numPlots) {
      # Get the i,j matrix positions of the regions that contain this subplot
      matchidx <- as.data.frame(which(layout == i, arr.ind = TRUE))
      print(plots[[i]], vp = viewport(layout.pos.row = matchidx$row,
                                      layout.pos.col = matchidx$col))
    }
  }
}

Subclass probability analisys (Attempt II)

Random Forest vs. world

botnet_prob_result %>% filter(TrueClass == 'Botnet') %>% group_by(subclass) %>% summarise(n=n(),sum_RF=sum(RamdomForest),sum_KNN=sum(KNN),sum_LR=sum(LogisticRegression),sum_NB=sum(NaiveBayes),sum_SVM=sum(SVM)) %>% arrange(desc(n))

subclass = botnet_prob_result %>% filter(TrueClass == 'Botnet') %>% group_by(subclass) %>% summarise(n = n()) %>% arrange(desc(n))
sc1 = botnet_prob_result %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < KNN) %>% group_by(subclass) %>% summarise(best_knn = n())
sc2 = botnet_prob_result %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < LogisticRegression) %>% group_by(subclass) %>% summarise(best_lr = n())
sc3 = botnet_prob_result %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < NaiveBayes) %>% group_by(subclass) %>% summarise(best_nb = n())
sc4 = botnet_prob_result %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < SVM) %>% group_by(subclass) %>% summarise(best_svm = n())
botnet_prob_result_table_subclass = inner_join(subclass,sc1,by="subclass") %>% inner_join(sc2,by="subclass") %>% inner_join(sc3,by="subclass") %>% inner_join(sc4,by="subclass")
botnet_prob_result_table_subclass

ports = botnet_prob_result %>% filter(TrueClass == 'Botnet') %>% group_by(port) %>% summarise(n = n()) %>% arrange(desc(n))
p1 = botnet_prob_result %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < KNN) %>% group_by(port) %>% summarise(best_knn = n())
p2 = botnet_prob_result %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < LogisticRegression) %>% group_by(port) %>% summarise(best_lr = n())
p3 = botnet_prob_result %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < NaiveBayes) %>% group_by(port) %>% summarise(best_nb = n())
p4 = botnet_prob_result %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < SVM) %>% group_by(port) %>% summarise(best_svm = n())
botnet_prob_result_table_port = inner_join(ports,p1,by="port") %>% inner_join(p2,by="port") %>% inner_join(p3,by="port") %>% inner_join(p4,by="port")
botnet_prob_result_table_port

protos = botnet_prob_result %>% filter(TrueClass == 'Botnet') %>% group_by(proto) %>% summarise(n = n()) %>% arrange(desc(n))
pr1 = botnet_prob_result %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < KNN) %>% group_by(proto) %>% summarise(best_knn = n())
pr2 = botnet_prob_result %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < LogisticRegression) %>% group_by(proto) %>% summarise(best_lr = n())
pr3 = botnet_prob_result %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < NaiveBayes) %>% group_by(proto) %>% summarise(best_nb = n())
pr4 = botnet_prob_result %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < SVM) %>% group_by(proto) %>% summarise(best_svm = n())
botnet_prob_result_table_proto = inner_join(protos,pr1,by="proto") %>% inner_join(pr2,by="proto") %>% inner_join(pr3,by="proto") %>% inner_join(pr4,by="proto")
botnet_prob_result_table_proto

botnet_prob_result_table_subclass$subclass = as.factor(botnet_prob_result_table_subclass$subclass)
data <- cbind(botnet_prob_result_table_subclass,id=seq(1:nrow(botnet_prob_result_table_subclass)))
data <- reshape2::melt(data[,-1],id.vars=c("id"))
data <- cbind(sub_class = botnet_prob_result_table_subclass$subclass,data)
p01 <- ggplot(data, aes(sub_class, value)) +
  geom_bar(aes(fill = variable), position = "dodge", stat="identity")
botnet_prob_result_table_port$port = as.factor(botnet_prob_result_table_port$port)
data <- cbind(botnet_prob_result_table_port,id=seq(1:nrow(botnet_prob_result_table_port)))
data <- reshape2::melt(data[,-1],id.vars=c("id"))
data <- cbind(ports = botnet_prob_result_table_port$port,data)
p02 <- ggplot(data, aes(ports, value)) +
  geom_bar(aes(fill = variable), position = "dodge", stat="identity")
botnet_prob_result_table_proto$proto = as.factor(botnet_prob_result_table_proto$proto)
data <- cbind(botnet_prob_result_table_proto,id=seq(1:nrow(botnet_prob_result_table_proto)))
data<-reshape2::melt(data[,-1],id.vars=c("id"))
data = cbind(protos = botnet_prob_result_table_proto$proto,data)
p03 <- ggplot(data, aes(protos, value)) +
  geom_bar(aes(fill = variable), position = "dodge", stat="identity")
############################### Filter by epsilon distance ########################################
epsilon = 0.09
sc1_e = botnet_prob_result %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < KNN) %>% filter((KNN - RamdomForest) > epsilon) %>% group_by(subclass) %>% summarise(best_knn = n())
sc2_e = botnet_prob_result %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < LogisticRegression) %>% filter((LogisticRegression - RamdomForest) > epsilon) %>% group_by(subclass) %>% summarise(best_lr = n())
sc3_e = botnet_prob_result %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < NaiveBayes) %>% filter((NaiveBayes - RamdomForest) > epsilon) %>% group_by(subclass) %>% summarise(best_nb = n())
sc4_e = botnet_prob_result %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < SVM) %>% filter((SVM - RamdomForest) > epsilon) %>% group_by(subclass) %>% summarise(best_svm = n())
botnet_prob_result_table_subclass_e = inner_join(subclass,sc1_e,by="subclass") %>% inner_join(sc2_e,by="subclass") %>% inner_join(sc3_e,by="subclass") %>% inner_join(sc4_e,by="subclass")
botnet_prob_result_table_subclass_e

p1_e = botnet_prob_result %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < KNN) %>% filter((KNN - RamdomForest) > epsilon) %>% group_by(port) %>% summarise(best_knn = n())
p2_e = botnet_prob_result %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < LogisticRegression) %>% filter((LogisticRegression - RamdomForest) > epsilon) %>% group_by(port) %>% summarise(best_lr = n())
p3_e = botnet_prob_result %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < NaiveBayes) %>% filter((NaiveBayes - RamdomForest) > epsilon) %>% group_by(port) %>% summarise(best_nb = n())
p4_e = botnet_prob_result %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < SVM) %>% filter((SVM - RamdomForest) > epsilon) %>% group_by(port) %>% summarise(best_svm = n())
botnet_prob_result_table_port_e = inner_join(ports,p1_e,by="port") %>% inner_join(p2_e,by="port") %>% inner_join(p3_e,by="port") %>% inner_join(p4_e,by="port")
botnet_prob_result_table_port_e

pr1_e = botnet_prob_result %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < KNN) %>% filter((KNN - RamdomForest) > epsilon) %>% group_by(proto) %>% summarise(best_knn = n())
pr2_e = botnet_prob_result %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < LogisticRegression) %>% filter((LogisticRegression - RamdomForest) > epsilon) %>% group_by(proto) %>% summarise(best_lr = n())
pr3_e = botnet_prob_result %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < NaiveBayes) %>% filter((NaiveBayes - RamdomForest) > epsilon) %>% group_by(proto) %>% summarise(best_nb = n())
pr4_e = botnet_prob_result %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < SVM) %>% filter((SVM - RamdomForest) > epsilon) %>% group_by(proto) %>% summarise(best_svm = n())
botnet_prob_result_table_proto_e = inner_join(protos,pr1_e,by="proto") %>% inner_join(pr2_e,by="proto") %>% inner_join(pr3_e,by="proto") %>% inner_join(pr4_e,by="proto")
botnet_prob_result_table_proto_e

botnet_prob_result_table_subclass_e$subclass = as.factor(botnet_prob_result_table_subclass_e$subclass)
data <- cbind(botnet_prob_result_table_subclass_e,id=seq(1:nrow(botnet_prob_result_table_subclass_e)))
data <- reshape2::melt(data[,-1],id.vars=c("id"))
data <- cbind(sub_class = botnet_prob_result_table_subclass_e$subclass,data)
p1 <- ggplot(data, aes(sub_class, value)) +
  geom_bar(aes(fill = variable), position = "dodge", stat="identity")
botnet_prob_result_table_port_e$port = as.factor(botnet_prob_result_table_port_e$port)
data <- cbind(botnet_prob_result_table_port_e,id=seq(1:nrow(botnet_prob_result_table_port_e)))
data <- reshape2::melt(data[,-1],id.vars=c("id"))
data <- cbind(ports = botnet_prob_result_table_port_e$port,data)
p2 <- ggplot(data, aes(ports, value)) +
  geom_bar(aes(fill = variable), position = "dodge", stat="identity")
botnet_prob_result_table_proto_e$proto = as.factor(botnet_prob_result_table_proto_e$proto)
data <- cbind(botnet_prob_result_table_proto_e,id=seq(1:nrow(botnet_prob_result_table_proto_e)))
data<-reshape2::melt(data[,-1],id.vars=c("id"))
data = cbind(protos = botnet_prob_result_table_proto_e$proto,data)
p3 <- ggplot(data, aes(protos, value)) +
  geom_bar(aes(fill = variable), position = "dodge", stat="identity")
multiplot(p02, p2, p03, p3,cols=2)

Clustering and PCA

kmeans_mod<-kmeans(testing[,1:10],centers = 10)
testing_cluster<-cbind(testing,cluster=kmeans_mod$cluster)
pca<-prcomp(testing[,c(-11,-12,-13,-14)], center = TRUE, scale. = TRUE) 
pca_testing<-data.frame(pca$x,class=testing_cluster$class,
                              subclass=testing_cluster$subclass,
                               cluster=testing_cluster$cluster
                               )
g<-ggplot(pca_testing,aes(x=PC1,y=PC2))+
  geom_jitter(aes(color=as.factor(subclass),text=cluster,shape=class))+
  #geom_point(aes(shape=asignacion),size=3)+
  ylab("PC1")+xlab("PC2")+
  theme_classic()+
#scale_shape_manual(values=c(8,6))+
   guides(color=FALSE,alpha=FALSE)

Ignoring unknown aesthetics: text

ggplotly(g)

We recommend that you use the dev version of ggplot2 with `ggplotly()`
Install it with: `devtools::install_github('hadley/ggplot2')`

Cluster probability analisys (Attempt II)

botnet_prob_cluster <- cbind(botnet_prob_result,cluster=kmeans_mod$cluster)
botnet_prob_cluster

epsilon = 0.09
cluster = botnet_prob_cluster %>% filter(TrueClass == 'Botnet') %>% group_by(cluster) %>% summarise(n = n()) %>% arrange(desc(n))
cl1 = botnet_prob_cluster %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < KNN) %>% group_by(cluster) %>% summarise(best_knn = n())
cl2 = botnet_prob_cluster %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < LogisticRegression) %>% group_by(cluster) %>% summarise(best_lr = n())
cl3 = botnet_prob_cluster %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < NaiveBayes) %>% group_by(cluster) %>% summarise(best_nb = n())
cl4 = botnet_prob_cluster %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < SVM) %>% group_by(cluster) %>% summarise(best_svm = n())
botnet_prob_result_table_cluster = inner_join(cluster,cl1,by="cluster") %>% inner_join(cl2,by="cluster") %>% inner_join(cl3,by="cluster") %>% inner_join(cl4,by="cluster")
botnet_prob_result_table_cluster

cl1_e = botnet_prob_cluster %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < KNN) %>% filter((KNN - RamdomForest) > epsilon) %>% group_by(cluster) %>% summarise(best_knn = n())
cl2_e = botnet_prob_cluster %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < LogisticRegression) %>% filter((LogisticRegression - RamdomForest) > epsilon) %>% group_by(cluster) %>% summarise(best_lr = n())
cl3_e = botnet_prob_cluster %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < NaiveBayes) %>% filter((NaiveBayes - RamdomForest) > epsilon) %>% group_by(cluster) %>% summarise(best_nb = n())
cl4_e = botnet_prob_cluster %>% filter(TrueClass == 'Botnet') %>% filter(RamdomForest < SVM) %>% filter((SVM - RamdomForest) > epsilon) %>% group_by(cluster) %>% summarise(best_svm = n())
botnet_prob_result_table_cluster_e = inner_join(cluster,cl1_e,by="cluster") %>% inner_join(cl2_e,by="cluster") %>% inner_join(cl3_e,by="cluster") %>% inner_join(cl4_e,by="cluster")
botnet_prob_result_table_cluster_e

cluster_bot = botnet_prob_cluster %>% filter(TrueClass == 'Botnet')

Bagging strategies

rfFit_by_port$finalModel

Call:
 randomForest(x = x, y = y, mtry = param$mtry) 
               Type of random forest: classification
                     Number of trees: 500
No. of variables tried at each split: 2

        OOB estimate of  error rate: 6.77%
Confusion matrix:
       Botnet Normal class.error
Botnet    628    232  0.26976744
Normal     68   3505  0.01903163

Bagging using adabag

training.bagging.cv$confusion

               Observed Class
Predicted Class Botnet Normal
         Botnet   4289    409
         Normal    839   4719

Training with some data set size.

output

gg <- ggplot(data = output)
gg + geom_line(aes(x = data_count, y = metric),color = 'red') + 
  labs(title="Random Forest through data training size", 
       #subtitle="Drawn from Long Data format", 
       caption="Source: CTU-13", 
       y="Accuracy", 
       color=NULL)

Normal Probabilistic table

#Normal probabilistic table
normal_prob_result = data.frame(testing$class, predsrfprobs$Normal, knnPredict$Normal, logicRPredict$Normal, naiveBayesPredict$Normal ,svmPredict$Normal)
names(normal_prob_result) = c('True Class','Ramdom Forest','KNN','Logistic Regression', 'Naive Bayes', 'Suport VM')
normal_prob_result